#!/bin/bash
#
# GNU General Public License v3.0; it is intended to be always in sync
# with the license term employed by the elbencho itself.
#
# Name :   graph_sweep.sh - a bash wrapper for mtelbencho.sh and optionally,
#                           gnuplot
#
# Author : Chin Fang <fangchin@zettar.com>
#
# Purpose: Make the graphing of storage sweep results simple.
#
# Remarks: 0. In the following, it is assumed that both mtelbencho.sh
#             and optionally gnuplot are in the $PATH. Nevertheless,
#             it always generate a gnuplot command file.
#          1. The wrapper enables you to select
#             1.0 The range to sweep (LOSF, Medium, Large, or Full. Default
#                 is a full sweep from 1KiB to 1TiB, as this provides the
#                 most comprehensive view of the storage system being
#                 benchmarked.
#             1.1 The number of times to repeat a desired sweep (default 3).
#          2. The format of the graph is set to svg. Nevertheless,
#             it's very easy to regenerate a plot based on your
#             preferences. See the descriptions for the two global
#             variable output and plot_file below.  The reason of the
#             choice is that very likely on a headless server, there
#             are no TrueType fonts installed, so svg is a safer
#             choice.  As an example, if the gnuplot term is set to
#             png, a popular graphics format, when running on a
#             server, the following warnings may appear:
#             gdImageStringFT: Could not find/open font while printing \
#             string 1048576x1KiB with font Verdana [...]
#          3. Since it runs mtelbencho.sh, so most options will be the
#             same as mtelbencho.sh's.  Please run mtelbencho.sh -h
#             and review the embedded Remarks in mtelbencho.sh for
#             more info.
#          4. As running mtelbencho.sh needs root privileges, so does
#             graph_sweep.sh. Run in a root shell or via sudo, except
#             in dry-run mode.
#          5. It uses the paste(1) command to form a result file
#             as the input to gnuplot.
#
# Usage : Assuming graph_sweep.sh is available from the $PATH,
#
#         cd src_data_dir;\
#         sudo graph_sweep.sh [-r s|m|l][-t N][-S fs_block_size]\
#         [-b block_size][-B][-N num_sweep][-o output_dir][-v][-T][-p][-n]
#
#         where src_data_dir is the directory that holds all test
#         datasets, N is the desired thread number to run elbencho,
#         num_sweep is the number of times to repeat a sweep,
#         output_dir is where the mtelbencho.sh's output will be
#         stored. Furthermore, four subdirectories, 'losf', 'medium',
#         'large', and 'full' are pre-created to prevent users from
#         accidentally having their results overwritten by different
#         runs for different ranges.  For other usage tips and
#         examples, on the CLI, type graph_sweep.sh -h for more info.
#
# References:
# 0. https://github.com/breuner/elbencho/tree/master/contrib/storage_sweep
# 1. http://www.gnuplot.info
# 2. https://man7.org/linux/man-pages/man1/paste.1.html
#
# A few global variables. Contrary to a myth popular among newbie
# programmers, if judiciously employed, global variables simplify
# programming :-S
#
# A terabyte is 1,000,000 megabytes. A megabytes is 1000000 bytes.
# So, a terabyte is 1000000 * 1000000 bytes. We require 
minimal_space=2000000000000
# preferred file descriptor limit for both unlimit -Hn and -Sn
fdlimit=262144
range_to_sweep=
threads=$(nproc)
src_data_dir=/var/tmp
fs_block_size=4
block_size=1
buffered=
num_sweep=3
output_dir=/var/tmp
traditional=
# Note that the gunplot command file is unconditionally generated.
push_button_plot=
today=$(date +%F)
out_file_base=$(hostname)_tests_"$today"
verbose=
dry_run=
#
# The two tools that this wrapper is used for.
#
mtelbencho=$(command -v mtelbencho.sh)
if [[ -z "$mtelbencho" ]]; then
    echo "mtelbencho.sh could not be found. Aborting!"
    exit
fi
# gnuplot is optional, unless -p (lower case p) is used
gnuplot=/usr/bin/gnuplot
#
# The sweep data file to be plotted using gnuplot. The CSV format is
# chosen such that people can still use a spreadsheet app should they
# choose :-S
#
sweep_csv="$output_dir"/sweep.csv
#
# You can customize the following and run gnuplot "$sweep_gplt" to
# regenerate a plot to your liking, with your choice of terminal type,
# font, font size, color, line width, line color and so on. It is a
# simple, almost self-explanatory text file. Much simpler than a
# spreadsheet application :-S
#
sweep_gplt="$output_dir"/sweep.gplt
#
# The final svg file for the sweep plot
#
sweep_svg="$output_dir"/sweep.svg
#
# All labels used for the X-axis of a plot are stored in the following
# array.
#
declare -a xlabels=('1048576x1KiB'   '1048576x2KiB'   '1048576x4KiB'
                    '1048576x8KiB'   '1048576x16KiB'  '1048576x32KiB'
                    '1048576x64KiB'  '1048576x128KiB' '1048576x256KiB'
                    '1048576x512KiB' '1048576x1MiB'   '524288x2MiB'
                    '262144x4MiB'    '131072x8MiB'    '65535x16MiB'
                    '32768x32MiB'    '16384x64MiB'    '8192x128MiB'
                    '4096x256MiB'    '2048x512MiB'    '1024x1GiB'
                    '512x2GiB'       '256x4GiB'       '128x8GiB'
                    '64x16GiB'       '32x32GiB'       '16x64GiB'
                    '8x128GiB'       '4x256GiB'       '2x512GiB'
                    '1x1TiB'
                   )

#
# The mtelbencho.sh uses individual files for the large-file range
# sweep. Their corresponding file descriptors could be all opened at
# the same time. Thus, the system default (often set to a modest 1024
# - 4096) may not be enough.  This can be raised with root
# privileges. Since this script is supposed to be run under root, so
# we can and will do it below temporarily - an application should
# NEVER touch the system's default settings!  In the following, both
# the hard and soft file descriptor limits are set to the same and
# adequate value per our experience.
#
set_max_open_file_descriptors()
{
    local hfdl
    hfdl=$(ulimit -Hn)
    local sfdl
    sfdl=$(ulimit -Sn)
    if (( "$hfdl < "$fdlimit )); then
        ulimit -Hn "$fdlimit"
    fi
    if (( "$sfdl < "$fdlimit )); then
        ulimit -Sn "$fdlimit"
    fi
}

check_app()
{
    local app=$1
    local msg
    
    if ! [[ -x "$app" ]]; then
        msg="$app is not installed."
        msg+="Please install it and then re-run this wizard. "
        msg+="Aborting now."
        echo "$msg"
        exit 1
    fi
}

check_dependencies()
{
    check_app "$mtelbencho"
    if [[ "$push_button_plot" ]]; then
        check_app "$gnuplot"
    fi
}

help()
{
    local help_str=''
    read -r -d '' help_str <<'EOF'
This script is a bash wrapper for mtelbencho.sh and optionally, gnuplot.

Its purpose is to make the graphing of storage sweep results simple. Note
that a storage sweep is best carried out on NVMe-based storage, whether
local or networked. HDD or SATA SSD-based storage may take too long.

Command line options:
    -h show this help
    -r the range to sweep, one of s: LOSF, m: medium, l: large.  If not
       specified, a full sweep is carried out to be consistent with 
       mtelbencho.sh's usage
    -t the number of threads to run elbencho. Default is auto-detect
    -s the path to the directory holding test datasets
    -S filesystem (aka fs) block size - positive integer; must be 2th
       power. File systems can have block size from 512 to (usually)
       65536 bytes. The default blocksize in most Linux filesystems is
       4096 bytes. So, the default is 4(K aka KiB). Note: the 512 bytes
       file system block size is deemed obsolete and is not supported!
    -b block size. Number of mebibytes to read/write in a single operation. 
       Default is 1M as used by elbencho
    -B use buffered I/O
    -N number of interations for a sweep (default: 3)
    -o the path to the directory where the results are stored
    -T tranditional output (use GB/s instead of Gbps; default Gbps)
    -p optional - generate plot using gnuplot directly on the server;
       this may be not acceptable at some sites which run the Minimal
       Install OS, or simply prohibited due to policy reasons. Nevertheless,
       a gnuplot command file is unconditionally generated. Running gnuplot
       on a workstation is more appropriate anyway. See also notes below
    -v verbose mode; to print out the test dataset, number of threads,
       and other settings
    -n dry-run; to print out tests that would have run

Furthermore, assuming

* The plot command file is named as sweep.gplt
* The input file (i.e. ifile) is named as sweep.csv
* The output file (i.e. ofile) is named as sweep.svg

then, if the gnuplot command file (default: sweep.gplt) has been
retrieved to the local computer, the following command will create
sweep.svg

$ gnuplot -e "ifile_arg='sweep.csv'; ofile_arg='sweep.svg'" sweep.gplt

Please replace sweep.csv, sweep.svg, and sweep.gplt if you have them
named differently.

The usage: 
$ sudo graph_sweep.sh [-h][-r range][-t numthreads][-s dirpath]\
  [-S fs_block_size][-b block_size][-B][-N num_sweep][-o output_dir]\
  [-v][-T][-p][-n]

Examples: 0. $ graph_sweep.sh -r s -t 56 -s /data/zettar/zx/src/sweep -b 16 \
               -o /var/tmp -p -v -n
          1. # graph_sweep.sh -r s -s /var/local/zettar/sweep -b 16 -p -v -N 1
          2. # graph_sweep.sh -r m -s /var/local/zettar/sweep -b 16 -p -v -N 1
          3. # graph_sweep.sh -r l -s /var/local/zettar/sweep -b 16 -p -v -N 1
          4. # graph_sweep.sh -s /var/local/zettar/sweep -b 16 -p -v -N 1
          5. # graph_sweep.sh -s /var/local/zettar/sweep -b 16 -v -N 1

In the above examples 
0. A dry run
1. Graph a single-run sweep for LOSF in one shot
2. Graph a single-run sweep for medium files in one shot
3. Graph a single-run sweep for large files in one shot
4. Graph a single-run full sweep for the full 1KiB-1TiB range in one shot
5. Generate gnuplot command file only. The rest is the same as 4 above
EOF
    echo "$help_str"
    exit 0
}

check_space_available()
{
    if ! [[ -d "$src_data_dir" ]]; then
	echo "$src_data_dir doesn't exist! Abort!"
	exit 1
    fi
    local sa
    # The df man page states that SIZE units default to 1024 bytes
    sa=$(df "$src_data_dir"|tail -1|awk '{ printf "%.0f", $4*1024 }')
    [[ "$verbose" ]] && echo "Space availability now is: $sa bytes."
    [[ "$verbose" ]] && echo "Minimal requirement is   : $minimal_space bytes."
    if [[ "$sa" -lt "$minimal_space" ]]; then
        local msg
        msg="Any storage sweep is carried out using hyperscale datasets, i.e.\n"
        msg+="each dataset's overall size >= 1TB or has >= 1M files or both.\n"
        msg+="As a precaution, 2TB free space is required to run a sweep.\n"
        msg+="Please free up more space and then re-run. Aborting now."
        echo -e "$msg"
        exit 1
    fi
}

print_iter()
{
    local i
    i=$1
    if [[ "$i" -eq "0" ]]; then
        echo "1st"
    elif [[ "$i" -eq "1" ]]; then
        echo "2nd"
    elif [[ "$i" -eq "2" ]]; then
        echo "3rd"
    else
        echo "$((i+1))th"
    fi
}

ensure_file_exists()
{
    local f
    f=$1
    if [[ -z "$dry_run" ]]; then
        if [[ -f "$f" ]] && [[ -s "$f" ]]; then 
            :
        else 
            echo "$f dose not exist or empty";
            exit 1
        fi      
    fi
}
    
run_mtelbencho()
{
    local out_file
    local cmd
    for ((i=0;i<num_sweep;i++));
    do
        echo "***> This is $(print_iter "$i") sweep"
        out_file="$out_file_base"_"$i".txt
        out_file="$output_dir/$out_file"
        cmd="$mtelbencho -t $threads -b $block_size -s $src_data_dir "
        [[ "$fs_block_size" -ne "4" ]] && cmd+="-S $fs_block_size "
        [[ "$buffered" ]] && cmd+="-B "
        [[ "$verbose" ]] && cmd+="-v "
        if [[ -n "$range_to_sweep" ]]; then
            cmd+="-r $range_to_sweep "
        fi
        if [[ "$dry_run" ]]; then
            echo "$cmd|tee $out_file"
        else
            $cmd|tee "$out_file"
        fi
    done
}

extract_results_for_plotting()
{
    local out_files
    out_files="${out_file_base}_*.txt"
    out_files="$output_dir/$out_files"
    local pasted_file
    pasted_file="$output_dir"/plot.dat
    local i=0
    local pfile
    # If Gbps is preferred, use the following conversion:
    # 0.008388608 = 1024*1024*8/(1000*1000*1000.),
    # i.e. Mebibyte*8bit/byte)/Gigabyte
    # If GB/s is preferred, use the following conversion:
    # 0.00104858 = 1024x1024/(1000*1000*1000).
    # i.e. Mebibytes/Gigabyte
    local conv
    if [[ "$traditional" ]]; then
	conv=0.00104858
    else
	conv=0.008388608
    fi
    for f in $out_files
    do
        pfile="$output_dir"/p"$i"
        grep Throughput < "$f"\
            |awk -v cf="$conv" '{printf "%.3f\n", (($4 + $5)*cf/2)}' \
                 > "$pfile"
        ensure_file_exists "$pfile"
        ((i++))
    done
    # Join the p"$i" files into a single output file, using the default \t
    # The [0-9] regex prevents hostnames with a leading p from messing up
    # the resulting file.
    paste "$output_dir"/p[0-9]* > "$pasted_file"
    ensure_file_exists "$pasted_file"
    # Create the final input file for gnuplot, stored in $sweep_csv, a global
    # variable :)
    local input
    input="$pasted_file"
    local i
    if [[ -z "$range_to_sweep" ]] || [[ "$range_to_sweep" == 's' ]] ; then
        i=0
    elif [[ "$range_to_sweep" == 'm' ]]; then
        i=10
    elif [[ "$range_to_sweep" == 'l' ]]; then
        i=20
    fi
    # The ending index doesn't pose a problem. The number of lines in
    # the input file automatically determines that.
    local mean_value 
    echo -e "Dataset,Mean-value" > "$sweep_csv"
    while IFS= read -r line
    do
        mean_value=$( echo "$line"|\
                          awk '{sum = 0; for (i = 1; i <= NF; i++) 
                          sum += $i; sum /= NF; print sum}')
        echo "${xlabels[$i]},$mean_value"
        ((i++))
    done < "$input" >>"$sweep_csv"
    ensure_file_exists "$sweep_csv"
    # Now we are ready to generate plot, either directly or optionally
    # retrieve the plot data thus generated to a workstation.
}

run_gnuplot()
{
    local gnuplot_settings=''
    local range
    if [[ "$range_to_sweep" == 's' ]]; then
        range='LOSF'
    elif [[ "$range_to_sweep" == 'm' ]]; then
        range='medium files'
    elif [[ "$range_to_sweep" == 'l' ]]; then
        range='large files'
    else
        range='full range'
    fi
    local speed
    if [[ "$traditional" ]]; then
	speed='GB/s'
    else
	speed='Gbps'
    fi
    local hn
    hn=$(hostname -s)
    read -r -d '' gnuplot_settings <<EOF
# The following gnuplot command file content is structured as follows:
#
# 1. Define paths and arguments
# 2. Set output method
# 3. Define graph content (titles, labels)
# 4. Set up basic graph structure (axes, grid)
# 5. Refine visual elements (line styles, grid styles)
# 6. Position supplementary information (legend)
# 7. Final visual tweaks (x-axis tics)
# 8. Plot the data
#
# Use default paths unless overridden
ifile = "/var/tmp/full/sweep.csv"
ofile = "/var/tmp/full/sweep.svg"

# Allow overriding through command-line arguments using gnuplot -e
if (exists("ifile_arg")) ifile = ifile_arg
if (exists("ofile_arg")) ofile = ofile_arg

# Set terminal options
#set terminal eps          # example. tested
#set terminal pdfcairo     # example. tested
set terminal svg enhanced background rgb 'white' # looks bad if no 
                                                 # background
# Set output file and data format
set output ofile
set datafile separator ',' # so that gnuplot understand we use a csv 
                           # file

# Set title and labels
set title "Storage sweep on $hn for $range over $num_sweep runs on $today"
set title font "Times Bold, 14"
set ylabel "Mean throughput ($speed)"
set ylabel font "Times Bold,10"

# Move up the xlabel by 3 characters to reduce the wasted white spaces
set xlabel 'Datasets' offset 0,3
set xlabel font "Times Bold,10"

# Configure axis and grid
set yrange [0:]
set style data line
set grid ytics xtics
set grid

# Set the line style (ls) for the grid
set style line 100 lt 1 lc rgb "grey" lw 0.5 
set grid ls 100 

# Configure the line styles
set style line 101 lw 3 lt rgb "#FF8C00"  # line color in drak orange
#set style line 102 lw 3 lt rgb "#0000FF" # example. line color in blue
#set style line 103 lw 4 lt rgb "#228B22" # example. line color in forestgreen

# Configure the legend (key)
set key autotitle columnhead
set key left top # legend placement
set key font "Verdana,8"
set key Left     # note the cap 'L'!  Left justify the key text
set key width -3 # shift the legend to the left

# Rotate and configure xtics
set xtics rotate # rotate labels on the x axis
set xtics font "Verdana,8"

# Plot the data
plot ifile using 2:xtic(1) ls 101  # use the line style 101 defined
                                   # earlier
EOF
    echo "$gnuplot_settings" > "$sweep_gplt"
    ensure_file_exists "$sweep_gplt" 
    # if the push_button_plot global variable is true, then the script will
    # also generate plot directly on the server to achieve a true push
    # button operation :) Otherwise, the user have the option to generate
    # a plot easily with gnuplot or another app, such as a spreadsheet.
    if [[ "$push_button_plot" ]]; then
        gnuplot "$sweep_gplt"
    fi
}

# A few more helper functions

run_as_root()
{
    local euid
    euid=$(id -u)
    if [[ "$euid" -ne 0 ]]; then
        local msg
        msg="You must be root to run mtelbencho.sh. Abort!"
        echo "$msg"
        exit 1
    fi
}

verify_range()
{
    local range_to_sweep
    range_to_sweep=$1
    local msg
    
    if [[ "$range_to_sweep" != [sml] ]]; then
        msg="Only s:LOSF, m:medium files, l:large files "
        msg+="allowed. Abort!"
        echo "$msg"
        exit 1
    fi    
}

verify_threads()
{
    # Note, threads is a global variable :)
    local msg
    
    if ! [[ "$threads" =~ ^[1-9][0-9]*$ ]]; then
        msg="threads must be a positive integer. Abort!"
        echo "$msg"
        exit 1
    fi    
}

verify_directory_exists()
{
    local dir
    dir=$1
    if [[ "$(cd "$dir")" -ne 0 ]]; then
        msg="$dir does not exist. Abort!"
        echo "$msg"
        exit 1
    fi
}

verify_block_size()
{
    # Note, block_size is a global variable :)  Also, this one differs subtly
    # from its counterpart in mtelbencho.sh!
    local msg
    if ! [[ "$block_size" =~ ^[1-9][0-9]*$ ]]; then
        msg="block_size must be specified in a positive integer. Abort!"
        echo "$msg"
        exit 1
    fi
}

verify_num_sweep()
{
    # Note, num_sweep is a global variable :)
    local msg
    
    if ! [[ "$num_sweep" =~ ^[1-9][0-9]*$ ]]; then
        msg="num_sweep must be specified in a positive integer. Abort!"
        echo "$msg"
        exit 1
    fi
}

mitigate_human_errors()
{
    #
    # It has been observed enough times that some new users run
    # successive sweep runs without putting the output of each sesion
    # into different output directories using the -o option, then
    # wondering why some graphs are missing etc.  OK, we will protect
    # such users by unconditionally pre-create the four subdirectories
    # {losf,medium,large,full} even the -o is used
    # 
    mkdir -p "$output_dir"/{losf,medium,large,full}
    if [[ "$range_to_sweep" == 's' ]]; then
        output_dir="$output_dir/losf"
    elif [[ "$range_to_sweep" == 'm' ]]; then
        output_dir="$output_dir/medium"
    elif [[ "$range_to_sweep" == 'l' ]]; then
        output_dir="$output_dir/large"
    else
        output_dir="$output_dir/full"
    fi
}

show_elbencho_version()
{
    local vs
    if ! command -v elbencho &> /dev/null
    then
        echo "elbencho could not be found. Abort!"
        exit 1
    fi
    vs=$(elbencho --version)
    echo "$vs"
}

show_installed_sweep_tools()
{
    echo "graph_sweep.sh is installed at $(command -v graph_sweep.sh)"
    echo "mtelbencho.sh is installed at $(command -v mtelbencho.sh)"
}

show_option_settings()
{
    echo "range_to_sweep  : $range_to_sweep"
    echo "threads         : $threads"
    echo "src_data_dir    : $src_data_dir"
    echo "fs_block_size   : $fs_block_size"    
    echo "block_size      : $block_size"
    echo "num_sweep       : $num_sweep"
    echo "output_dir      : $output_dir"
    echo "buffered        : $buffered"
    echo "verbose         : $verbose"
    echo "traditional     : $traditional"
    echo "push_button_plot: $push_button_plot"
    echo "dry_run         : $dry_run"
    echo "sweep_csv       : $sweep_csv"
    echo "sweep_gplt      : $sweep_gplt"
    echo "sweep_svg       : $sweep_svg"
}

is_power_of_two()
{
    if ! ((fs_block_size > 0 &&
                (fs_block_size & (fs_block_size - 1)) == 0 )); then
        msg="fs_block_size must be specified in a positive integer "
        msg+="in 2th power.  Abort!"
        echo "$msg"
        exit 1
    fi
}

show_test_duration()
{
    local total_test_time
    total_test_time=$1
    local t_hsec
    local t_hor
    local t_min
    local t_sec
    
    t_hor=$((total_test_time / 3600))
    if [[ "$t_hor" -gt 0 ]]; then
        t_hsec=$((t_hor * 3600))
        t_hsec=$((total_test_time - t_hsec))
        t_min=$((t_hsec / 60))
        t_sec=$((total_test_time % 60))
        echo "Total test time: ${t_hor}h:${t_min}m:${t_sec}s"
    else
        t_min=$((total_test_time / 60))
        t_sec=$((total_test_time % 60)) 
        echo "Total test time: ${t_min}m:${t_sec}s"
    fi
}

# main()
{
    begin_test=$(date +"%s")
    while getopts ":hr:t:s:S:b:N:o:BvTnp" opt; do
        case $opt in
            h)  help
                ;;
            r)  range_to_sweep=$OPTARG
                verify_range "$range_to_sweep"
                ;;          
            t)  threads=$OPTARG
                verify_threads
                ;;
            s)  src_data_dir=$OPTARG # dir check to be done later
                ;;
            S)  fs_block_size=$OPTARG
                is_power_of_two
                ;;
            b)  block_size=$OPTARG
                verify_block_size
                ;;
            N)  num_sweep=$OPTARG
                verify_num_sweep
                ;;
            o)  output_dir=$OPTARG   # dir check to be done later
                ;;
            B)  buffered=1
                ;;          
            v)  verbose=1
                ;;
	    T)  traditional=1
		;;
            p)  push_button_plot=1
                ;;
            n)  dry_run=1
                ;;
            *)  echo "Error: invalid option given!"
                exit 2
                ;;
        esac
    done

    mitigate_human_errors
    sweep_csv="$output_dir"/sweep.csv
    sweep_gplt="$output_dir"/sweep.gplt
    sweep_svg="$output_dir"/sweep.svg
    [[ "$verbose" ]] && show_elbencho_version
    [[ "$verbose" ]] && show_installed_sweep_tools
    [[ "$verbose" ]] && show_option_settings
    check_space_available
    [[ -z "$dry_run" ]] && verify_directory_exists "$src_data_dir"
    [[ -z "$dry_run" ]] && verify_directory_exists "$output_dir"
    [[ -z "$dry_run" ]] && set_max_open_file_descriptors
    [[ -z "$dry_run" ]] && check_dependencies
    [[ -z "$dry_run" ]] && run_as_root
    [[ "$verbose" ]] && echo "===> Getting ready to sweep..."
    run_mtelbencho
    [[ "$verbose" ]] && echo "===> Sweeps done. Extracting results..."
    [[ -z "$dry_run" ]] && extract_results_for_plotting
    [[ "$verbose" ]] && echo "===> Results extracted. Plotting..."
    [[ -z "$dry_run" ]] && run_gnuplot
    [[ "$verbose" ]] && echo "===> $0 all done :)"
    end_test=$(date +"%s")
    total_test_time=$((end_test-begin_test))
    [[ "$verbose" ]] && show_test_duration "$total_test_time"
    exit 0
} # end of main
