Commit c3e983dd authored by Jack Doerner's avatar Jack Doerner

Added benchmark scripts

parent ef69f105
......@@ -41,6 +41,12 @@ Project Organization
Source for this project is divided into two directories: `src` contains code for the primary library, while `tests` contains code for tests and benchmarks. The library will be compiled to `build/lib/liback.a`, and all testing and benchmarking binaries are found in `build/tests`.
Reproducing Results
=====
For the purpose of reproducing the results we report in our paper, we provide a suite of benchmark scripts in the `tools/bench` directory. Each script must be executed on one machine as a server, and on another as a client. Scripts will run as server by default, and will output data and summaries to the `benchmark_results` directory. Adding the `-c <address>` flag will cause the script to run as a client and connect to the specified server.
Running Tests and Benchmarks Manually
=====
......
#!/bin/bash
mkdir -p ../../benchmark_results/binarysearch/results
mkdir -p ../../benchmark_results/binarysearch/samples
OUTPUT_FILE_NAME=circuit_`date +%s`_$$.txt
SAMPLE_FILE=../../benchmark_results/binarysearch/samples/$OUTPUT_FILE_NAME
RESULT_FILE=../../benchmark_results/binarysearch/results/$OUTPUT_FILE_NAME
touch $SAMPLE_FILE
touch $RESULT_FILE
set -e
CLIENT=false
BENCH_PROG="../../build/tests/bench_bst"
BENCH_PROG_ARGS="-e 32768 -o circuit "
SEARCHES=(1 32 1024)
ITERS=(1 1 1)
while getopts ":c:" opt; do
case $opt in
c)
BENCH_PROG_ARGS+=" -c $OPTARG "
CLIENT=true
;;
\?)
echo "Invalid option: -$OPTARG" >&2
exit 1
;;
:)
echo "Option -$OPTARG requires an argument." >&2
exit 1
;;
esac
done
for ((II=0; II<${#SEARCHES[*]}; II++));
do
COMMAND="$BENCH_PROG $BENCH_PROG_ARGS -s ${SEARCHES[II]} -i ${ITERS[II]}"
echo "COMMAND: $COMMAND"
echo "COMMAND: $COMMAND" >> $RESULT_FILE
eval "$COMMAND" >> $SAMPLE_FILE 2>> $RESULT_FILE
if [ "$CLIENT" = true ] ; then
sleep 5
fi
done
\ No newline at end of file
#!/bin/bash
mkdir -p ../../benchmark_results/binarysearch/results
mkdir -p ../../benchmark_results/binarysearch/samples
OUTPUT_FILE_NAME=fssl_`date +%s`_$$.txt
SAMPLE_FILE=../../benchmark_results/binarysearch/samples/$OUTPUT_FILE_NAME
RESULT_FILE=../../benchmark_results/binarysearch/results/$OUTPUT_FILE_NAME
touch $SAMPLE_FILE
touch $RESULT_FILE
set -e
CLIENT=false
BENCH_PROG="../../build/tests/bench_bs"
BENCH_PROG_ARGS="-e 32768 -o fssl "
SEARCHES=(1 32 1024)
ITERS=(30 30 30)
while getopts ":c:" opt; do
case $opt in
c)
BENCH_PROG_ARGS+=" -c $OPTARG "
CLIENT=true
;;
\?)
echo "Invalid option: -$OPTARG" >&2
exit 1
;;
:)
echo "Option -$OPTARG requires an argument." >&2
exit 1
;;
esac
done
for ((II=0; II<${#SEARCHES[*]}; II++));
do
COMMAND="OMP_THREAD_LIMIT=4 $BENCH_PROG $BENCH_PROG_ARGS -s ${SEARCHES[II]} -i ${ITERS[II]}"
echo "COMMAND: $COMMAND"
echo "COMMAND: $COMMAND" >> $RESULT_FILE
eval "$COMMAND" >> $SAMPLE_FILE 2>> $RESULT_FILE
if [ "$CLIENT" = true ] ; then
sleep 5
fi
done
\ No newline at end of file
#!/bin/bash
mkdir -p ../../benchmark_results/binarysearch/results
mkdir -p ../../benchmark_results/binarysearch/samples
OUTPUT_FILE_NAME=fssl_`date +%s`_$$.txt
SAMPLE_FILE=../../benchmark_results/binarysearch/samples/$OUTPUT_FILE_NAME
RESULT_FILE=../../benchmark_results/binarysearch/results/$OUTPUT_FILE_NAME
touch $SAMPLE_FILE
touch $RESULT_FILE
set -e
CLIENT=false
BENCH_PROG="../../build/tests/bench_bs"
BENCH_PROG_ARGS="-e 1048576 -o fssl "
SEARCHES=(1 32 1024)
ITERS=(3 3 3)
while getopts ":c:" opt; do
case $opt in
c)
BENCH_PROG_ARGS+=" -c $OPTARG "
CLIENT=true
;;
\?)
echo "Invalid option: -$OPTARG" >&2
exit 1
;;
:)
echo "Option -$OPTARG requires an argument." >&2
exit 1
;;
esac
done
for ((II=0; II<${#SEARCHES[*]}; II++));
do
COMMAND="OMP_THREAD_LIMIT=4 $BENCH_PROG $BENCH_PROG_ARGS -s ${SEARCHES[II]} -i ${ITERS[II]}"
echo "COMMAND: $COMMAND"
echo "COMMAND: $COMMAND" >> $RESULT_FILE
eval "$COMMAND" >> $SAMPLE_FILE 2>> $RESULT_FILE
if [ "$CLIENT" = true ] ; then
sleep 5
fi
done
\ No newline at end of file
#!/bin/bash
mkdir -p ../../benchmark_results/binarysearch/results
mkdir -p ../../benchmark_results/binarysearch/samples
OUTPUT_FILE_NAME=fssl_`date +%s`_$$.txt
SAMPLE_FILE=../../benchmark_results/binarysearch/samples/$OUTPUT_FILE_NAME
RESULT_FILE=../../benchmark_results/binarysearch/results/$OUTPUT_FILE_NAME
touch $SAMPLE_FILE
touch $RESULT_FILE
set -e
CLIENT=false
BENCH_PROG="../../build/tests/bench_bs"
BENCH_PROG_ARGS="-e 33554432 -o fssl "
SEARCHES=(1)
ITERS=(3 3 3)
while getopts ":c:" opt; do
case $opt in
c)
BENCH_PROG_ARGS+=" -c $OPTARG "
CLIENT=true
;;
\?)
echo "Invalid option: -$OPTARG" >&2
exit 1
;;
:)
echo "Option -$OPTARG requires an argument." >&2
exit 1
;;
esac
done
for ((II=0; II<${#SEARCHES[*]}; II++));
do
COMMAND="$BENCH_PROG $BENCH_PROG_ARGS -s ${SEARCHES[II]} -i ${ITERS[II]}"
echo "COMMAND: $COMMAND"
echo "COMMAND: $COMMAND" >> $RESULT_FILE
eval "$COMMAND" >> $SAMPLE_FILE 2>> $RESULT_FILE
if [ "$CLIENT" = true ] ; then
sleep 5
fi
done
\ No newline at end of file
#!/bin/bash
mkdir -p ../../benchmark_results/binarysearch/results
mkdir -p ../../benchmark_results/binarysearch/samples
OUTPUT_FILE_NAME=fssl_cprg_`date +%s`_$$.txt
SAMPLE_FILE=../../benchmark_results/binarysearch/samples/$OUTPUT_FILE_NAME
RESULT_FILE=../../benchmark_results/binarysearch/results/$OUTPUT_FILE_NAME
touch $SAMPLE_FILE
touch $RESULT_FILE
set -e
CLIENT=false
BENCH_PROG="../../build/tests/bench_bs"
BENCH_PROG_ARGS="-e 32768 -o fssl_cprg "
SEARCHES=(1 32 1024)
ITERS=(30 30 30)
while getopts ":c:" opt; do
case $opt in
c)
BENCH_PROG_ARGS+=" -c $OPTARG "
CLIENT=true
;;
\?)
echo "Invalid option: -$OPTARG" >&2
exit 1
;;
:)
echo "Option -$OPTARG requires an argument." >&2
exit 1
;;
esac
done
for ((II=0; II<${#SEARCHES[*]}; II++));
do
COMMAND="OMP_THREAD_LIMIT=2 $BENCH_PROG $BENCH_PROG_ARGS -s ${SEARCHES[II]} -i ${ITERS[II]}"
echo "COMMAND: $COMMAND"
echo "COMMAND: $COMMAND" >> $RESULT_FILE
eval "$COMMAND" >> $SAMPLE_FILE 2>> $RESULT_FILE
if [ "$CLIENT" = true ] ; then
sleep 5
fi
done
\ No newline at end of file
#!/bin/bash
mkdir -p ../../benchmark_results/binarysearch/results
mkdir -p ../../benchmark_results/binarysearch/samples
OUTPUT_FILE_NAME=fssl_cprg_`date +%s`_$$.txt
SAMPLE_FILE=../../benchmark_results/binarysearch/samples/$OUTPUT_FILE_NAME
RESULT_FILE=../../benchmark_results/binarysearch/results/$OUTPUT_FILE_NAME
touch $SAMPLE_FILE
touch $RESULT_FILE
set -e
CLIENT=false
BENCH_PROG="../../build/tests/bench_bs"
BENCH_PROG_ARGS="-e 1048576 -o fssl_cprg "
SEARCHES=(1 32 1024)
ITERS=(3 3 3)
while getopts ":c:" opt; do
case $opt in
c)
BENCH_PROG_ARGS+=" -c $OPTARG "
CLIENT=true
;;
\?)
echo "Invalid option: -$OPTARG" >&2
exit 1
;;
:)
echo "Option -$OPTARG requires an argument." >&2
exit 1
;;
esac
done
for ((II=0; II<${#SEARCHES[*]}; II++));
do
COMMAND="OMP_THREAD_LIMIT=2 $BENCH_PROG $BENCH_PROG_ARGS -s ${SEARCHES[II]} -i ${ITERS[II]}"
echo "COMMAND: $COMMAND"
echo "COMMAND: $COMMAND" >> $RESULT_FILE
eval "$COMMAND" >> $SAMPLE_FILE 2>> $RESULT_FILE
if [ "$CLIENT" = true ] ; then
sleep 5
fi
done
\ No newline at end of file
#!/bin/bash
mkdir -p ../../benchmark_results/binarysearch/results
mkdir -p ../../benchmark_results/binarysearch/samples
OUTPUT_FILE_NAME=fssl_cprg_`date +%s`_$$.txt
SAMPLE_FILE=../../benchmark_results/binarysearch/samples/$OUTPUT_FILE_NAME
RESULT_FILE=../../benchmark_results/binarysearch/results/$OUTPUT_FILE_NAME
touch $SAMPLE_FILE
touch $RESULT_FILE
set -e
CLIENT=false
BENCH_PROG="../../build/tests/bench_bs"
BENCH_PROG_ARGS="-e 33554432 -o fssl_cprg "
SEARCHES=(1)
ITERS=(3 3 3)
while getopts ":c:" opt; do
case $opt in
c)
BENCH_PROG_ARGS+=" -c $OPTARG "
CLIENT=true
;;
\?)
echo "Invalid option: -$OPTARG" >&2
exit 1
;;
:)
echo "Option -$OPTARG requires an argument." >&2
exit 1
;;
esac
done
for ((II=0; II<${#SEARCHES[*]}; II++));
do
COMMAND="$BENCH_PROG $BENCH_PROG_ARGS -s ${SEARCHES[II]} -i ${ITERS[II]}"
echo "COMMAND: $COMMAND"
echo "COMMAND: $COMMAND" >> $RESULT_FILE
eval "$COMMAND" >> $SAMPLE_FILE 2>> $RESULT_FILE
if [ "$CLIENT" = true ] ; then
sleep 5
fi
done
\ No newline at end of file
#!/bin/bash
mkdir -p ../../benchmark_results/binarysearch/results
mkdir -p ../../benchmark_results/binarysearch/samples
OUTPUT_FILE_NAME=sqrt_`date +%s`_$$.txt
SAMPLE_FILE=../../benchmark_results/binarysearch/samples/$OUTPUT_FILE_NAME
RESULT_FILE=../../benchmark_results/binarysearch/results/$OUTPUT_FILE_NAME
touch $SAMPLE_FILE
touch $RESULT_FILE
set -e
CLIENT=false
BENCH_PROG="../../build/tests/bench_bst"
BENCH_PROG_ARGS="-e 32768 -o sqrt "
SEARCHES=(1 32 1024)
ITERS=(30 30 30)
while getopts ":c:" opt; do
case $opt in
c)
BENCH_PROG_ARGS+=" -c $OPTARG "
CLIENT=true
;;
\?)
echo "Invalid option: -$OPTARG" >&2
exit 1
;;
:)
echo "Option -$OPTARG requires an argument." >&2
exit 1
;;
esac
done
for ((II=0; II<${#SEARCHES[*]}; II++));
do
COMMAND="$BENCH_PROG $BENCH_PROG_ARGS -s ${SEARCHES[II]} -i ${ITERS[II]}"
echo "COMMAND: $COMMAND"
echo "COMMAND: $COMMAND" >> $RESULT_FILE
eval "$COMMAND" >> $SAMPLE_FILE 2>> $RESULT_FILE
if [ "$CLIENT" = true ] ; then
sleep 5
fi
done
\ No newline at end of file
#!/bin/bash
mkdir -p ../../benchmark_results/binarysearch/results
mkdir -p ../../benchmark_results/binarysearch/samples
OUTPUT_FILE_NAME=sqrt_`date +%s`_$$.txt
SAMPLE_FILE=../../benchmark_results/binarysearch/samples/$OUTPUT_FILE_NAME
RESULT_FILE=../../benchmark_results/binarysearch/results/$OUTPUT_FILE_NAME
touch $SAMPLE_FILE
touch $RESULT_FILE
set -e
CLIENT=false
BENCH_PROG="../../build/tests/bench_bst"
BENCH_PROG_ARGS="-e 1048576 -o sqrt "
SEARCHES=(1 32 1024)
ITERS=(3 3 3)
while getopts ":c:" opt; do
case $opt in
c)
BENCH_PROG_ARGS+=" -c $OPTARG "
CLIENT=true
;;
\?)
echo "Invalid option: -$OPTARG" >&2
exit 1
;;
:)
echo "Option -$OPTARG requires an argument." >&2
exit 1
;;
esac
done
for ((II=0; II<${#SEARCHES[*]}; II++));
do
COMMAND="$BENCH_PROG $BENCH_PROG_ARGS -s ${SEARCHES[II]} -i ${ITERS[II]}"
echo "COMMAND: $COMMAND"
echo "COMMAND: $COMMAND" >> $RESULT_FILE
eval "$COMMAND" >> $SAMPLE_FILE 2>> $RESULT_FILE
if [ "$CLIENT" = true ] ; then
sleep 5
fi
done
\ No newline at end of file
#!/bin/bash
mkdir -p ../../benchmark_results/access/samples
mkdir -p ../../benchmark_results/access/results
OUTPUT_FILE_NAME=circuit_`date +%s`_$$.txt
SAMPLE_FILE=../../benchmark_results/access/samples/$OUTPUT_FILE_NAME
RESULT_FILE=../../benchmark_results/access/results/$OUTPUT_FILE_NAME
touch $SAMPLE_FILE
touch $RESULT_FILE
set -e
CLIENT=false
BENCH_PROG="../../build/tests/bench_oram"
BENCH_PROG_ARGS="-s 1 -o circuit "
ELCTS=(32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 131072 262144 524288 1048576 2097152 4194304 8388608 16777216 33554432 67108864)
ITERS=(100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100)
while getopts ":c:" opt; do
case $opt in
c)
BENCH_PROG_ARGS+=" -c $OPTARG "
CLIENT=true
;;
\?)
echo "Invalid option: -$OPTARG" >&2
exit 1
;;
:)
echo "Option -$OPTARG requires an argument." >&2
exit 1
;;
esac
done
for ((II=0; II<${#ELCTS[*]}; II++));
do
COMMAND="$BENCH_PROG $BENCH_PROG_ARGS -e ${ELCTS[II]} -i ${ITERS[II]}"
echo "COMMAND: $COMMAND"
echo "COMMAND: $COMMAND" >> $RESULT_FILE
eval "$COMMAND" >> $SAMPLE_FILE 2>> $RESULT_FILE
if [ "$CLIENT" = true ] ; then
sleep 5
fi
done
\ No newline at end of file
#!/bin/bash
mkdir -p ../../benchmark_results/init/results
mkdir -p ../../benchmark_results/init/samples
OUTPUT_FILE_NAME=circuit_`date +%s`_$$.txt
SAMPLE_FILE=../../benchmark_results/init/samples/$OUTPUT_FILE_NAME
RESULT_FILE=../../benchmark_results/init/results/$OUTPUT_FILE_NAME
touch $SAMPLE_FILE
touch $RESULT_FILE
set -e
CLIENT=false
BENCH_PROG="../../build/tests/bench_oram_init"
BENCH_PROG_ARGS="-s 1 -o circuit "
ELCTS=(32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 131072 262144)
ITERS=(30 30 30 30 30 30 30 30 30 3 3 3 3 3)
while getopts ":c:" opt; do
case $opt in
c)
BENCH_PROG_ARGS+=" -c $OPTARG "
CLIENT=true
;;
\?)
echo "Invalid option: -$OPTARG" >&2
exit 1
;;
:)
echo "Option -$OPTARG requires an argument." >&2
exit 1
;;
esac
done
for ((II=0; II<${#ELCTS[*]}; II++));
do
COMMAND="$BENCH_PROG $BENCH_PROG_ARGS -e ${ELCTS[II]} -i ${ITERS[II]}"
echo "COMMAND: $COMMAND"
echo "COMMAND: $COMMAND" >> $RESULT_FILE
eval "$COMMAND" >> $SAMPLE_FILE 2>> $RESULT_FILE
if [ "$CLIENT" = true ] ; then
sleep 5
fi
done
\ No newline at end of file
#!/bin/bash
mkdir -p ../../benchmark_results/access/results
mkdir -p ../../benchmark_results/access/samples
OUTPUT_FILE_NAME=fssl_`date +%s`_$$.txt
SAMPLE_FILE=../../benchmark_results/access/samples/$OUTPUT_FILE_NAME
RESULT_FILE=../../benchmark_results/access/results/$OUTPUT_FILE_NAME
touch $SAMPLE_FILE
touch $RESULT_FILE
set -e
CLIENT=false
BENCH_PROG="../../build/tests/bench_oram_write"
BENCH_PROG_ARGS="-s 1 -o fssl "
ELCTS=(67108864 134217728 268435456 536870912 1073741824 2147483648 4294967296 8589934592 17179869184)
ITERS=(1024 1449 2048 2897 4096 5793 8192 11586 16384)
while getopts ":c:" opt; do
case $opt in
c)
BENCH_PROG_ARGS+=" -c $OPTARG "
CLIENT=true
;;
\?)
echo "Invalid option: -$OPTARG" >&2
exit 1
;;
:)
echo "Option -$OPTARG requires an argument." >&2
exit 1
;;
esac
done
for ((II=0; II<${#ELCTS[*]}; II++));
do
COMMAND="$BENCH_PROG $BENCH_PROG_ARGS -e ${ELCTS[II]} -i ${ITERS[II]}"
echo "COMMAND: $COMMAND"
echo "COMMAND: $COMMAND" >> $RESULT_FILE
eval "$COMMAND" >> $SAMPLE_FILE 2>> $RESULT_FILE
if [ "$CLIENT" = true ] ; then
sleep 5
fi
done
\ No newline at end of file
#!/bin/bash
mkdir -p ../../benchmark_results/access_limited/results
mkdir -p ../../benchmark_results/access_limited/samples
OUTPUT_FILE_NAME=fssl_`date +%s`_$$.txt
SAMPLE_FILE=../../benchmark_results/access_limited/samples/$OUTPUT_FILE_NAME
RESULT_FILE=../../benchmark_results/access_limited/results/$OUTPUT_FILE_NAME
touch $SAMPLE_FILE
touch $RESULT_FILE
set -e
CLIENT=false
BENCH_PROG="../../build/tests/bench_oram_write"
BENCH_PROG_ARGS="-s 1 -o fssl "
ELCTS=(32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 131072 262144 524288 1048576 2097152 4194304 8126940 16777216 33554432)
ITERS=(100 100 100 100 100 100 102 104 108 112 115 128 138 128 182 128 182 256 384 512 725)
TCTS=(2 2 2 2 2 2 4 4 4 4 4 4 4 4 4 4 4 4 8 8 8)
while getopts ":c:" opt; do
case $opt in
c)
BENCH_PROG_ARGS+=" -c $OPTARG "
CLIENT=true
;;
\?)
echo "Invalid option: -$OPTARG" >&2
exit 1
;;
:)
echo "Option -$OPTARG requires an argument." >&2
exit 1
;;
esac
done
for ((II=0; II<${#ELCTS[*]}; II++));
do
COMMAND="OMP_THREAD_LIMIT=${TCTS[II]} $BENCH_PROG $BENCH_PROG_ARGS -e ${ELCTS[II]} -i ${ITERS[II]}"
echo "COMMAND: $COMMAND"
echo "COMMAND: $COMMAND" >> $RESULT_FILE
eval "$COMMAND" >> $SAMPLE_FILE 2>> $RESULT_FILE
if [ "$CLIENT" = true ] ; then
sleep 5
fi
done
\ No newline at end of file
#!/bin/bash
mkdir -p ../../benchmark_results/access_threads/results
mkdir -p ../../benchmark_results/access_threads/samples
OUTPUT_FILE_NAME=fssl_`date +%s`_$$.txt
SAMPLE_FILE=../../benchmark_results/access_threads/samples/$OUTPUT_FILE_NAME
RESULT_FILE=../../benchmark_results/access_threads/results/$OUTPUT_FILE_NAME
touch $SAMPLE_FILE
touch $RESULT_FILE
set -e
CLIENT=false
BENCH_PROG="../../build/tests/bench_oram_write"
BENCH_PROG_ARGS=" -s 1 -o fssl "
ELS=(1024 1024 1024 1024 1024 32768 32768 32768 32768 32768 1048576 1048576 1048576 1048576 1048576 33554432 33554432 33554432 33554432 33554432 1073741824 1073741824 1073741824 1073741824 1073741824)
ITERS=(32 32 32 32 32 182 182 182 182 182 256 256 256 256 256 725 725 725 725 725 4096 4096 4096 4096 4096)
TCTS=(1 2 4 8 16 1 2 4 8 16 1 2 4 8 16 1 2 4 8 16 1 2 4 8 16)
while getopts ":c:" opt; do
case $opt in
c)
BENCH_PROG_ARGS+=" -c $OPTARG "
CLIENT=true
;;
\?)
echo "Invalid option: -$OPTARG" >&2
exit 1
;;
:)
echo "Option -$OPTARG requires an argument." >&2
exit 1
;;
esac
done
for ((II=0; II<${#TCTS[*]}; II++));
do
COMMAND="OMP_THREAD_LIMIT=${TCTS[II]} $BENCH_PROG $BENCH_PROG_ARGS -e ${ELS[II]} -i ${ITERS[II]}"
echo "COMMAND: $COMMAND"
echo "COMMAND: $COMMAND" >> $RESULT_FILE
eval "$COMMAND" >> $SAMPLE_FILE 2>> $RESULT_FILE
if [ "$CLIENT" = true ] ; then
sleep 5
fi
done
\ No newline at end of file
#!/bin/bash
mkdir -p ../../benchmark_results/access/results
mkdir -p ../../benchmark_results/access/samples
OUTPUT_FILE_NAME=fssl_cprg_`date +%s`_$$.txt
SAMPLE_FILE=../../benchmark_results/access/samples/$OUTPUT_FILE_NAME
RESULT_FILE=../../benchmark_results/access/results/$OUTPUT_FILE_NAME
touch $SAMPLE_FILE
touch $RESULT_FILE
set -e
CLIENT=false
BENCH_PROG="../../build/tests/bench_oram_write"
BENCH_PROG_ARGS="-s 1 -o fssl_cprg "
ELCTS=(16777216 33554432 67108864 134217728 268435456 536870912 1073741824 2147483648 4294967296 8589934592 17179869184)
ITERS=(512 725 1024 1449 2048 2897 4096 5793 8192 11586 16384)
while getopts ":c:" opt; do