#!/bin/bash
#
# Generate chart from elbencho csv result file via gnuplot and/or Excel.


CSVFILE=""                   # input file
OUTIMGFILE=""                # output file for chart image
OUTXLSXFILE=""               # output file for excel sheet
OUTIMGBGCOLOR=""             # output image rgb background (e.g. "#ffffff" for white)
TMPOPSTRING="OPERATION"      # string in tmp file name to replace by actual op (e.g. "READ")
TMPOPFILE="/tmp/elbencho-plot-${TMPOPSTRING}.csv" # tmp files for each requested op (e.g. "READ")
TERMINALDEFAULT="qt"         # default gnuplot terminal type (overridden by image file output)
CHARTSIZE=""                 # width,height of chart (to override gnuplot default)
CHARTKEYPOS="top center"     # position of line descriptions within chart area
CHARTBORDERFLAGS="15"        # bitwise flags for axis (1=bottom, 2=left, 4=top, 8=right)
LINEWIDTH="3"                # width of graph lines
POINTSIZE="0.5"              # scale of data points between graph lines (relative to font size)
POINTBOX="1"                 # scale of box that creates space between data points and graph lines
FONTSIZE=""                  # font size (to override gnuplot default)
CHARTTITLE=""                # chart headline
CHARTXTITLE=""               # x-axis title
CHARTYTITLE=""               # left-hand y-axis title
CHARTY2TITLE=""              # right-hand y-axis title
USEBARS=""                   # "1" for bar chart instead of default line chart
CSVCOLUMNS_X=()              # user-given columns for x-axis tick labels
CSVCOLUMNSIDX_X=()           # discovered column indices for x-axis labels
CSVCOLUMNS_Y_LEFT=()         # user-given columns for left y-axis. format: "columnname[:opfilter]"
CSVCOLUMNSIDX_Y_LEFT=()      # discovered column indices for left y-axis
CSVCOLUMNS_Y_RIGHT=()        # user-given columns for right y-axis. format: "columnname[:opfilter]"
CSVCOLUMNSIDX_Y_RIGHT=()     # dicsovered column indices for right y-axis
XTICLABELROTDEGREES="0"      # x-axis tick label rotation degrees
NUMCSVCOLUMNS="0"            # discovered number of columns in csv input file
CSVOPCOLUMN="operation"      # title of operation column in csv input file
LINECOLORS=("#26afd0" "#2a9df4" "#fa8f6f" "#fadb6f" "#ff6961" "#a8cce8" "#d5aaff" "#54fffb")


# Print usage info and exit
usage()
{
   echo "Generate chart from elbencho csv result file via gnuplot."
   echo
   echo "Usage: $(basename -- $0) [OPTIONS] <CSVFILE>"
   echo
   echo "Main Options:"
   echo "  CSVFILE            Path to elbencho results csv file."
   echo "  -c                 List all available columns in csv file and exit."
   echo "  -o                 List all available operations in csv file and exit."
   echo "  -x COL             Csv file column to use for x-axis labels."
   echo "                     This option can be used multiple times for combined labels."
   echo "                     (Hint: Use \"-c\" to see avaiable columns.)"
   echo "  -y COL[:OP]        Csv file column to use for graph on left-hand y-axis. OP is"
   echo "                     the operation in case your csv file contains multiple ops,"
   echo "                     e.g. read and write results. (Hint: Use \"-o\" to see"
   echo "                     available operations in csv file.)"
   echo "                     This option can be used multiple times for multiple graphs."
   echo "  -Y COL[:OP]        Csv file column to use for graph on right-hand y-axis."
   echo "                     This option can be used multiple times for multiple graphs."
   echo
   echo "Output Options:"
   echo "  --imgfile PATH     Path to output image file (svg, png, pdf)."
   echo "                     If no output option is given, a gnuplot window is shown."
   echo "  --excel PATH       Path to output Excel file with embedded chart."
   echo
   echo "Misc Options:"
   echo "  --bars             Generate bar chart. Default is line chart."
   echo "  --chartsize W,H    Chart width and height in pixels."
   echo "                     Exception: Size for pdf image file is in inches."
   echo "  --fontsize NUM     Font size."
   echo "  --imgbg RGB        Image background color instead of transparent background."
   echo "                     Example: \"#ffffff\" for white background."
   echo "  --keypos STRING    Position of key with line descriptions within chart area."
   echo "                     Default is \"top center\"."
   echo "  --linecolors LIST  Space-separated list of RGB color codes for chart lines"
   echo "                     in quotes to appear as single argument, e.g.:"
   echo "                     \"#ff0000 #00ff00 #0000ff\" (including quotes)."
   echo "  --linewidth NUM    Line width."
   echo "  --title STRING     Set chart title."
   echo "  --xrot NUM         Rotate x-axis tick labels by given number of degrees."
   echo "  --xtitle STRING    Set title for x-axis."
   echo "  --ytitle STRING    Set title for left-hand y-axis."
   echo "  --Ytitle STRING    Set title for right-hand y-axis."
   echo
   echo "Elbencho Example:"
   echo "  As the basis for a graph, you need to have multiple results in the csv file,"
   echo "  e.g. like this to generate results for different block sizes:"
   echo "    $ for block in 4k 64k 256k 1m; do \\"
   echo "        elbencho -w -r -b \$block -s10g --direct \\"
   echo "          --csvfile elbencho-results.csv /data/testfile; \\"
   echo "      done"
   echo
   echo "Usage Examples:"
   echo "  1) List available columns in csv file \"elbencho-results.csv\":"
   echo "     $ $(basename -- $0) -c elbencho-results.csv"
   echo
   echo "  2) Generate read throughput (left y-axis) and IOPS (right y-axis) graphs for"
   echo "     different blocks sizes from elbencho-results.csv:"
   echo "     $ $(basename -- $0) -x \"block size\" -y \"MiB/s [last]:READ\" \\"
   echo "         -Y \"IOPS [last]:READ\" elbencho-results.csv"
   echo
   echo "  3) Generate the same chart but save it to an Excel file:"
   echo "     $ $(basename -- $0) -x \"block size\" -y \"MiB/s [last]:READ\" \\"
   echo "         -Y \"IOPS [last]:READ\" --excel my-chart.xlsx elbencho-results.csv"

   exit 1
}

# Parse command line arguments
parse_args()
{
   local i
   local list_csv_columns_only="0"
   local list_csv_ops_only="0"

   while [ $# -gt 0 ]; do
      case $1 in

         # options that don't require an argument

         --bars)
            USEBARS="1"
            ;;
         -c)
            list_csv_columns_only="1"
            ;;
         -h|--help)
            usage
            exit 0
            ;;
         -o)
            list_csv_ops_only="1"
            ;;

         # options that require an argument (i.e. additional shift)

        --borderflags)
           CHARTBORDERFLAGS="$2"
           shift
           ;;
        --chartsize)
           CHARTSIZE="$2"
           shift
           ;;
        --excel)
           OUTXLSXFILE="$2"
           shift
           ;;
        --fontsize)
           FONTSIZE="$2"
           shift
           ;;
        --imgfile)
           OUTIMGFILE="$2"
           shift
           ;;
        --imgbg)
           OUTIMGBGCOLOR="$2"
           shift
           ;;
        --keypos)
           CHARTKEYPOS="$2"
           shift
           ;;
        --linecolors)
           LINECOLORS=($2)
           shift
           ;;
        --linewidth)
           LINEWIDTH="$2"
           shift
           ;;
        --pointbox)
           POINTBOX="$2"
           shift
           ;;
        --pointsize)
           POINTSIZE="$2"
           shift
           ;;
        --terminaltype)
           TERMINALDEFAULT="$2"
           shift
           ;;
        --title)
           CHARTTITLE="$2"
           shift
           ;;
        -x)
           CSVCOLUMNS_X+=("$2")
           shift
           ;;
        --xrot)
           XTICLABELROTDEGREES="$2"
           shift
           ;;
        --xtitle)
           CHARTXTITLE="$2"
           shift
           ;;
        -y)
           CSVCOLUMNS_Y_LEFT+=("$2")
           shift
           ;;
        -Y)
           CSVCOLUMNS_Y_RIGHT+=("$2")
           shift
           ;;
        --ytitle)
           CHARTYTITLE="$2"
           shift
           ;;
        --Ytitle)
           CHARTY2TITLE="$2"
           shift
           ;;

        # other options starting with dash are invalid

        (-*)
           echo "ERROR: Unrecognized option: $1" 1>&2
           exit 1
           ;;

        # csv input file is last argument

        (*)
           CSVFILE="$1"
           shift
           break
           ;;
      esac

      shift # shift to next argument
   done

   # check the basics to be able to run special commands

   if [ $# -gt 0 ]; then
      echo "ERROR: Options are not allowed after csv filename: $@" 1>&2
      exit 1
   fi

   if [ -z "$CSVFILE" ]; then
      echo "ERROR: Input csv file undefined."
      exit 1
   fi

   if [ ! -e "$CSVFILE" ]; then
      echo "ERROR: Input file not found: $CSVFILE" 1>&2
      exit 1
   fi

   if [ $(cat "$CSVFILE" | wc -l) -lt 2 ]; then
      echo "ERROR: Input file needs at least 2 lines (one headline and one data line): $CSVFILE" 1>&2
      exit 1
   fi

   # init global NUMCSVCOLUMNS and sanity-check number of columns
   # (required for special commands below)

   NUMCSVCOLUMNS=$(head -n1 "$CSVFILE" | awk -F, '{print NF}')

   if [ "$NUMCSVCOLUMNS" -lt 2 ]; then
      echo "ERROR: Input csv file needs at least 2 columns (one for x- and one for left y-axis)." 1>&2
      exit 1
   fi

   # run special commands (instead of graph generation)
   # (note: we run them here because csv file is not defined at options parsing time)

   if [ "$list_csv_columns_only" -eq 1 ]; then
      list_csv_columns || exit 1
      exit 0
   fi

   if [ "$list_csv_ops_only" -eq 1 ]; then
      list_csv_ops || exit 1
      exit 0
   fi

   # ensure we have at least one x-axis tick description
   if [ "${#CSVCOLUMNS_X[@]}" -eq 0 ]; then
      echo "ERROR: X-axis csv file column undefined." 1>&2
      exit 1
   fi

   # ensure we have at least one left-hand y-axis defined
   if [ "${#CSVCOLUMNS_Y_LEFT[@]}" -eq 0 ]; then
      echo "ERROR: Left y-axis csv file column undefined." 1>&2
      exit 1
   fi

   # check op filters

   for (( i=0; i < ${#CSVCOLUMNS_Y_LEFT[@]}; i++ )); do
      $(check_op_filter_rows_exist_or_exit "${CSVCOLUMNS_Y_LEFT[$i]}") || exit 1
   done

   for (( i=0; i < ${#CSVCOLUMNS_Y_RIGHT[@]}; i++ )); do
      $(check_op_filter_rows_exist_or_exit "${CSVCOLUMNS_Y_RIGHT[$i]}") || exit 1
   done
}

# Check if gnuplot is installed and exit if not.
find_gnuplot_or_exit()
{
   if ! which gnuplot >/dev/null 2>&1; then
      echo "ERROR: gnuplot command not found. Install gnuplot and try again."
      exit 1
   fi
}

# Check if python3 and required libs are installed and exit if not.
find_python_or_exit()
{
   if ! which python3 >/dev/null 2>&1; then
      echo "ERROR: python3 command not found. Install python3 and try again."
      exit 1
   fi
   if ! python3 -c "import pandas" >/dev/null 2>&1; then
      echo "ERROR: Python pandas library not found. Install with 'pip install pandas'."
      exit 1
   fi
   if ! python3 -c "import xlsxwriter" >/dev/null 2>&1; then
      echo "ERROR: Python xlsxwriter library not found. Install with 'pip install xlsxwriter'."
      exit 1
   fi
}

# List csv file columns, one per line.
list_csv_columns()
{
   local i

   for (( i=1; i <= $NUMCSVCOLUMNS; i++ )); do
      CURRENT_COLUMN=$(head -n1 "$CSVFILE" | cut -d, -f${i} )
      echo "Column $i: \"$CURRENT_COLUMN\""
   done
}

# List csv file operations, one per line.
list_csv_ops()
{
   local i
   local csvopcolumnidx

   if ! csvopcolumnidx+=$(find_column_index "${CSVOPCOLUMN}"); then
      echo "ERROR: Column not found: ${CSVCOLUMNS_X[$i]}" >&2
      exit 1
   fi

   echo "Available operations in csv file:"

   tail -n+2 "$CSVFILE" | cut "-d," -f ${csvopcolumnidx} | sort -u || exit 1
}

# Create copy of csv input file in /tmp for given op name.
# Do nothing if given op name is empty string.
#
# $1: op name (e.g. "READ")
create_op_file_copy()
{
   local opfilter="$1"

   # do nothing if opfilter empty
   if [ -z "$opfilter" ]; then
      return
   fi

   # op filter set, so create corresponding tmp file

   local tmpfile="${TMPOPFILE/$TMPOPSTRING/$opfilter}"

   head -n1 "$CSVFILE" > "$tmpfile" || exit 1
   tail -n+2 "$CSVFILE" | grep ",$opfilter," >> "$tmpfile" || exit 1
}

# Create copies of csv input file in /tmp containing only the selected operation types.
# Note: This might create the same op file multiple times (for each graph line where the same op
# type is given), but that doesn't hurt.
create_op_files()
{
   local opfilter
   local i

   for (( i=0; i < ${#CSVCOLUMNS_Y_LEFT[@]}; i++ )); do
      opfilter=$(get_op_filter "${CSVCOLUMNS_Y_LEFT[$i]}")
      create_op_file_copy "$opfilter" || exit 1
   done

   for (( i=0; i < ${#CSVCOLUMNS_Y_RIGHT[@]}; i++ )); do
      opfilter=$(get_op_filter "${CSVCOLUMNS_Y_RIGHT[$i]}")
      create_op_file_copy "$opfilter" || exit 1
   done
}

# Get input file for given column[:op].
# Will return/echo the general csv input file if no op defined and tmp op file path otherwise.
#
# $1: "column:op" string as in (CSVCOLUMNS_Y_LEFT[$X])
get_input_file()
{
   local opfilter
   local tmpfile

   opfilter=$(get_op_filter "$1")

   # return general csv input file if opfilter empty
   if [ -z "$opfilter" ]; then
      echo "$CSVFILE"
      return
   fi

   # op filter set, so return corresponding tmp file

   tmpfile="${TMPOPFILE/$TMPOPSTRING/$opfilter}"
   echo "$tmpfile"
}

# Check if a certain operation filter is defined for a graph.
# Returns/echos op filter if set, empty string otherwise.
#
# $1: "column:op" string as in (CSVCOLUMNS_Y_LEFT[$X])
get_op_filter()
{
   # note: cut fields work only if delimiter exists in string

   # check if delimiter found, so we can use cut.
   # (otherwise there is no op filter, so nothing to echo)
   if $(echo "$1" | grep -q ":"); then
      echo "$1" | cut "-d:" -f 2
   fi
}

# Remove operation filter from column definition and return only the column name.
#
# $1: "column:op" string as in (CSVCOLUMNS_Y_LEFT[$X])
get_column_name()
{
   # note: cut fields work only if delimiter exists in string

   # check if delimiter found, so we can use cut.
   # (otherwise there is no op filter, so echo full string)
   if $(echo "$1" | grep -q ":"); then
      echo "$1" | cut "-d:" -f 1
   else
      echo "$1"
   fi
}

# Return number of rows found in csv file for a given op filter
#
# $1: op filter string (e.g. "READ")
count_op_filter_lines()
{
   grep ",$1," "$CSVFILE" | wc -l
}

# Check if csv rows are found for a given op filter.
#
# $1: "column:op" string as in (CSVCOLUMNS_Y_LEFT[$X])
check_op_filter_rows_exist_or_exit()
{
   local opfilter
   local numlines

   opfilter=$(get_op_filter "$1")

   if [ -z "$opfilter" ]; then
      return; # nothing to do if no op filter set
   fi

   numlines=$(count_op_filter_lines "$opfilter")
   if [ "$numlines" -eq 0 ]; then
      echo "ERROR: No lines found in csv file for given operation filter: \"$opfilter\"" 1>&2
      exit 1
   fi
}

# Find column index in CSV file
#
# $1: Column text to search
# Return: CSV column index
find_column_index()
{
   local COLUMNTEXT=$(get_column_name "$1")
   local CURRENT_COLUMN
   local FOUND_COLUMN_IDX
   local i

   for (( i=1; i <= $NUMCSVCOLUMNS; i++ )); do
      CURRENT_COLUMN=$(head -n1 $CSVFILE | cut -d, -f${i} )
      if [ "$CURRENT_COLUMN" = "$COLUMNTEXT" ]; then
         FOUND_COLUMN_IDX=$i
         break
      fi
   done

   if [ -z "$FOUND_COLUMN_IDX" ]; then
      return 1
   fi

   echo "$FOUND_COLUMN_IDX"
}

# Set x- and left/right y-axis column index array variables.
# Exit 1 if column not found.
set_column_indices()
{
   local i

   for (( i=0; i < ${#CSVCOLUMNS_X[@]}; i++ )); do
      if ! CSVCOLUMNSIDX_X+=($(find_column_index "${CSVCOLUMNS_X[$i]}")); then
         echo "ERROR: Column not found: ${CSVCOLUMNS_X[$i]}" >&2
         exit 1
      fi
   done

   for (( i=0; i < ${#CSVCOLUMNS_Y_LEFT[@]}; i++ )); do
      if ! CSVCOLUMNSIDX_Y_LEFT+=($(find_column_index "${CSVCOLUMNS_Y_LEFT[$i]}")); then
         echo "ERROR: Column not found: ${CSVCOLUMNS_Y_LEFT[$i]}" >&2
         exit 1
      fi
   done

   for (( i=0; i < ${#CSVCOLUMNS_Y_RIGHT[@]}; i++ )); do
      if ! CSVCOLUMNSIDX_Y_RIGHT+=($(find_column_index "${CSVCOLUMNS_Y_RIGHT[$i]}")); then
         echo "ERROR: Column not found: ${CSVCOLUMNS_Y_RIGHT[$i]}" >&2
         exit 1
      fi
   done
}

# Generate x-axis tic labels gnuplot subcommand.
# This uses inputfile or first left-hand y-axis graph line as basis, because the general csv input
# file might have other lines before the actual graph data.
generate_gnuplot_xlabels()
{
   local IFS=$'\n' # yes, the dollar sign is intended here
   local i
   local labelidx=0
   local inputfile

   # use inputfile or first left-hand y-axis graph line as basis
   inputfile=$(get_input_file "${CSVCOLUMNS_Y_LEFT[0]}")

   # awk_cmd looks like this in the end:
   #    NF {print \$9 \" \" \$25 \" \"}

   awk_cmd="NF {print "

   for (( i=0; i < ${#CSVCOLUMNSIDX_X[@]}; i++ )); do
      if [ $i -gt 0 ]; then
         awk_cmd+=' "\\@" '
      fi
      awk_cmd+="\$${CSVCOLUMNSIDX_X[$i]}"
   done

   awk_cmd+="}"

   # xtics_cmd looks like this in the end:
   #    set xtics ('col1_txt1 col2_txt1' 0, 'col1_txt2 col2_txt2' 1, 'col1_txt3 col2_txt3' 2, )
   xtics_cmd="set xtics ("

   for label in $(tail -n+2 "$inputfile" | awk "-F," "$awk_cmd"); do
      xtics_cmd+="'$label' ${labelidx}, "
      ((labelidx++))
   done

   xtics_cmd+=") rotate by $XTICLABELROTDEGREES right"

   echo "$xtics_cmd"
}

# Get title for given line.
#
# $1: "column:op" string as in (CSVCOLUMNS_Y_LEFT[$X])
get_line_title()
{
   local opfilter
   local columnlabel

   opfilter=$(get_op_filter "$1")
   columnlabel=$(get_column_name "$1")

   if [ -z "$opfilter" ]; then
      # no opfilter set, so just return given column string
      echo "$1"
   else
      echo "$opfilter $columnlabel"
   fi
}

# Generate gnuplot terminal device command. The "terminal" can be a window or an image file if
# OUTIMGFILE is set.
generate_gnuplot_terminal_cmd()
{
   local filename
   local extension
   local term_font=""
   local chart_size=""
   local transparent="transparent" # separate var because not all terms support transparent keyword
   local imgbgcolor=""

   [ -z "$CHARTSIZE" ] || chart_size="size ${CHARTSIZE}"
   [ -z "$FONTSIZE" ] || term_font="font ',${FONTSIZE}'"

   # Graphical window if no output image file is requested
   if [ -z "$OUTIMGFILE" ]; then
      echo "set terminal ${TERMINALDEFAULT} ${chart_size} ${term_font} enhanced"
      return
   fi

   filename=$(basename -- "$OUTIMGFILE")

   # "##*." deletes longest match of "*." from filename
   extension="${filename##*.}"

   # set user-defined background color instead of default transparent
   if [ ! -z "$OUTIMGBGCOLOR" ]; then
      transparent="" # we have bg color, so no transparent bg
      imgbgcolor="background '$OUTIMGBGCOLOR'"
   fi

   case "$extension" in
      pdf)
         # note: pdfcairo only supports size in inches and cm (not in pixels).
         # note: pdfcairo doesn't support transparent background.
         echo "set terminal pdfcairo enhanced color ${chart_size} ${term_font} $imgbgcolor"
         echo "set output '$OUTIMGFILE'"
         ;;
      png)
         echo "set terminal pngcairo enhanced color $transparent $imgbgcolor ${chart_size} ${term_font}"
         echo "set output '$OUTIMGFILE'"
         ;;
      svg)
         echo "set terminal svg dynamic $imgbgcolor ${chart_size} ${term_font}"
         echo "set output '$OUTIMGFILE'"
         ;;
      *)
         echo "ERROR: Unknown output image file extension: $OUTIMGFILE" 1>&2
         exit 1
         ;;
   esac
}

# Generate commands for gnuplot
generate_gnuplot_cmd()
{
   local plot_cmd=""
   local graphtype="with linespoints"
   local LINECOLOROFFSET="101" # offset for gnuplot line definition number
   local linecoloridx="0"
   local inputfile
   local linetitle
   local i

   # set terminal device to graphical window or output image file
   generate_gnuplot_terminal_cmd || exit 1

   # define the basics
   echo "set datafile separator ','"
   echo "set key autotitle columnhead"
   echo "set key $CHARTKEYPOS"
   echo "set tics out"
   echo "set xtics nomirror"
   echo "set ytics nomirror"
   echo "set y2tics"
   echo 'set yrange [0:*]'
   echo 'set y2range [0:*]'
   echo "set autoscale yfixmin"
   echo "set autoscale y2fixmin"
   echo "set style line 100 lt 1 lc rgb \"grey\" lw 0.5" # grid line style
   echo "set pointintervalbox $POINTBOX" # small gaps between graph line and data points
   echo "set border $CHARTBORDERFLAGS" # bitwise (1=bottom, 2=left, 4=top, 8=right)

   # define line style (colors, width etc)
   linecoloridx="0"
   for (( i=0; i < ${#LINECOLORS[@]}; i++ )); do
      linecoldefnum="$(($linecoloridx + $LINECOLOROFFSET))"
      # note: point type (pt) 7 is filled circle
      echo "set style line $linecoldefnum lw $LINEWIDTH lt rgb \"${LINECOLORS[$linecoloridx]}\" pt 7 pi -1 ps $POINTSIZE"
      ((linecoloridx++))
   done

   # set labels for x-axis ticks
   echo "$(generate_gnuplot_xlabels)"

   # x-axis tic labels right aligned if right rotated, left aligned if left rotated, else centered
   if [ $XTICLABELROTDEGREES -gt 0 ]; then
      echo "set xtics right"
   elif [ $XTICLABELROTDEGREES -lt 0 ]; then
      echo "set xtics left"
   else
      echo "set xtics center"
   fi

   # init bar chart details if requested
   [ -z "$USEBARS" ] || echo "set boxwidth 0.7"
   [ -z "$USEBARS" ] || echo "set style fill solid"
   [ -z "$USEBARS" ] || echo "set style data histogram"
   [ -z "$USEBARS" ] || echo "set style histogram cluster gap 1"
   [ -z "$USEBARS" ] || graphtype=""

   # set chart and axis titles
   [ -z "$CHARTTITLE" ] || echo "set title '${CHARTTITLE}'"
   [ -z "$CHARTXTITLE" ] || echo "set xlabel \"${CHARTXTITLE}\""
   [ -z "$CHARTYTITLE" ] || echo "set ylabel \"${CHARTYTITLE}\""
   [ -z "$CHARTY2TITLE" ] || echo "set y2label \"${CHARTY2TITLE}\""

   # generate plot command (separate for left-hand and right-hand y-axis)...

   linecoloridx="0"

   plot_cmd="plot "

   # left-hand y-axis
   for (( i=0; i < ${#CSVCOLUMNSIDX_Y_LEFT[@]}; i++ )); do
      linetitle=$(get_line_title "${CSVCOLUMNS_Y_LEFT[$i]}")
      linecoldefnum="$(($linecoloridx + $LINECOLOROFFSET))"
      inputfile=$(get_input_file "${CSVCOLUMNS_Y_LEFT[$i]}")

      plot_cmd+="\"${inputfile}\" using ${CSVCOLUMNSIDX_Y_LEFT[$i]} title \"${linetitle}\" $graphtype ls $linecoldefnum axis x1y1, "

      ((linecoloridx++))
   done

   # right-hand y-axis
   for (( i=0; i < ${#CSVCOLUMNSIDX_Y_RIGHT[@]}; i++ )); do
      linetitle=$(get_line_title "${CSVCOLUMNS_Y_RIGHT[$i]}")
      linecoldefnum="$(($linecoloridx + $LINECOLOROFFSET))"
      inputfile=$(get_input_file "${CSVCOLUMNS_Y_RIGHT[$i]}")

      plot_cmd+="\"${inputfile}\" using ${CSVCOLUMNSIDX_Y_RIGHT[$i]} title \"${linetitle}\" $graphtype ls $linecoldefnum axis x1y2, "

     ((linecoloridx++))
   done

   echo "$plot_cmd"
}

# Generate Excel chart by invoking an embedded python script
generate_excel_chart() {
    python3 - "$OUTXLSXFILE" \
            "$CSVFILE" \
            "$USEBARS" \
            "$CHARTTITLE" \
            "$CHARTXTITLE" \
            "$CHARTYTITLE" \
            "$CHARTY2TITLE" \
            "$CHARTKEYPOS" \
            "$XTICLABELROTDEGREES" \
            "${#CSVCOLUMNS_X[@]}" "${CSVCOLUMNS_X[@]}" \
            "${#CSVCOLUMNS_Y_LEFT[@]}" "${CSVCOLUMNS_Y_LEFT[@]}" \
            "${#CSVCOLUMNS_Y_RIGHT[@]}" "${CSVCOLUMNS_Y_RIGHT[@]}" \
            "${#LINECOLORS[@]}" "${LINECOLORS[@]}" \
    <<EOF
import sys
import pandas as pd

def parse_col_op(col_op_string):
    """Splits a 'column:operation' string into a (column, operation) tuple."""
    if ':' in col_op_string:
        parts = col_op_string.split(':', 1)
        return parts[0], parts[1]
    else:
        return col_op_string, None

def main():
    # --- Argument Parsing from stdin ---
    args = sys.argv[1:]
    out_xlsx_file = args.pop(0)
    csv_file = args.pop(0)
    use_bars = args.pop(0)
    chart_title = args.pop(0)
    chart_xtitle = args.pop(0)
    chart_ytitle = args.pop(0)
    chart_y2title = args.pop(0)
    chart_keypos = args.pop(0)
    xtic_label_rot_degrees = args.pop(0)

    # Reconstruct arrays
    num_x = int(args.pop(0))
    cols_x = [args.pop(0) for _ in range(num_x)]
    num_y_left = int(args.pop(0))
    cols_y_left = [args.pop(0) for _ in range(num_y_left)]
    num_y_right = int(args.pop(0))
    cols_y_right = [args.pop(0) for _ in range(num_y_right)]
    num_colors = int(args.pop(0))
    line_colors = [args.pop(0) for _ in range(num_colors)]

    # --- Data Processing with Pandas ---
    try:
        df = pd.read_csv(csv_file)
    except FileNotFoundError:
        print(f"ERROR: CSV file not found: {csv_file}", file=sys.stderr)
        sys.exit(1)

    series_data = {}
    all_y_cols_op = cols_y_left + cols_y_right

    for y_col_op in all_y_cols_op:
        col_name, op_filter = parse_col_op(y_col_op)
        filtered_df = df[df['operation'] == op_filter].copy() if op_filter else df.copy()
        filtered_df.reset_index(drop=True, inplace=True)
        series_data[y_col_op] = filtered_df[col_name]

    first_y_col_op = cols_y_left[0]
    _, first_op_filter = parse_col_op(first_y_col_op)
    
    base_df = df[df['operation'] == first_op_filter].copy() if first_op_filter else df.copy()
    base_df.reset_index(drop=True, inplace=True)
    x_labels = base_df[cols_x].astype(str).agg('@'.join, axis=1)

    output_df = pd.DataFrame(series_data)
    
    x_axis_label_name = '@'.join(cols_x)
    output_df.insert(0, x_axis_label_name, x_labels)

    # --- Excel Chart Generation with XlsxWriter ---
    writer = pd.ExcelWriter(out_xlsx_file, engine='xlsxwriter')
    output_df.to_excel(writer, sheet_name='ChartData', index=False)

    workbook = writer.book
    worksheet = writer.sheets['ChartData']

    num_rows = len(output_df)
    color_idx = 0

    # Create the chart. For clustered bar charts with a secondary axis, we must
    # create two separate chart objects and combine them.
    if use_bars:
        # Create a primary chart for the left y-axis series
        chart = workbook.add_chart({'type': 'column', 'subtype': 'clustered'})
    else:
        # Line charts don't have this issue and can be a single object
        chart = workbook.add_chart({'type': 'line'})

    # Add left y-axis series to the primary chart
    for i, col_op in enumerate(cols_y_left):
        col_name, op_filter = parse_col_op(col_op)
        series_name = f"{op_filter} {col_name}" if op_filter else col_name
        
        chart.add_series({
            'name':       series_name,
            'categories': ['ChartData', 1, 0, num_rows, 0],
            'values':     ['ChartData', 1, i + 1, num_rows, i + 1],
            'line':       {'color': line_colors[color_idx]},
            'fill':       {'color': line_colors[color_idx]},
        })
        color_idx += 1

    # Add right y-axis series.
    if cols_y_right:
        if use_bars:
            # For bar charts, create a secondary chart object to combine
            secondary_chart = workbook.add_chart({'type': 'column', 'subtype': 'clustered'})
            
            # Add series to the secondary chart, making sure to mark y2_axis
            for i, col_op in enumerate(cols_y_right):
                col_name, op_filter = parse_col_op(col_op)
                series_name = f"{op_filter} {col_name}" if op_filter else col_name
                secondary_chart.add_series({
                    'name':       series_name,
                    'categories': ['ChartData', 1, 0, num_rows, 0],
                    'values':     ['ChartData', 1, len(cols_y_left) + i + 1, num_rows, len(cols_y_left) + i + 1],
                    'y2_axis':    True,
                    'line':       {'color': line_colors[color_idx]},
                    'fill':       {'color': line_colors[color_idx]},
                })
                color_idx += 1
            
            # Combine the primary and secondary charts
            chart.combine(secondary_chart)
        else:
            # For line charts, just add series to the original chart
            for i, col_op in enumerate(cols_y_right):
                col_name, op_filter = parse_col_op(col_op)
                series_name = f"{op_filter} {col_name}" if op_filter else col_name
                chart.add_series({
                    'name':       series_name,
                    'categories': ['ChartData', 1, 0, num_rows, 0],
                    'values':     ['ChartData', 1, len(cols_y_left) + i + 1, num_rows, len(cols_y_left) + i + 1],
                    'y2_axis':    True,
                    'line':       {'color': line_colors[color_idx]},
                })
                color_idx += 1

    # --- Chart Formatting ---
    chart.set_title({'name': chart_title})
    chart.set_x_axis({'name': chart_xtitle, 'num_font': {'rotation': int(xtic_label_rot_degrees)}})
    chart.set_y_axis({'name': chart_ytitle, 'major_gridlines': {'visible': True}})
    if cols_y_right:
        chart.set_y2_axis({'name': chart_y2title})
    
    legend_position = 'top'
    if 'right' in chart_keypos: legend_position = 'right'
    if 'left' in chart_keypos: legend_position = 'left'
    if 'bottom' in chart_keypos: legend_position = 'bottom'
    chart.set_legend({'position': legend_position})
    
    if use_bars:
        chart.set_style(10)

    worksheet.insert_chart('K2', chart, {'x_scale': 2, 'y_scale': 2})
    writer.close()

if __name__ == "__main__":
    main()
EOF
}


##################### end of function definitions / start of main commands ###################


parse_args "$@"

# Determine which outputs are requested
DO_GNUPLOT=0
DO_EXCEL=0

# Default to gnuplot window if no output file is specified
if [ -z "$OUTIMGFILE" ] && [ -z "$OUTXLSXFILE" ]; then
    DO_GNUPLOT=1
fi
if [ -n "$OUTIMGFILE" ]; then
    DO_GNUPLOT=1
fi
if [ -n "$OUTXLSXFILE" ]; then
    DO_EXCEL=1
fi

# Check for required tools
if [ "$DO_GNUPLOT" -eq 1 ]; then
    find_gnuplot_or_exit
fi
if [ "$DO_EXCEL" -eq 1 ]; then
    find_python_or_exit
fi

set_column_indices || exit 1

if [ -z "${CSVCOLUMNS_X[0]}" ]; then
   echo "ERROR: X-axis csv file column undefined." 1>&2
   exit 1
fi

if [ -z "${CSVCOLUMNS_Y_LEFT[0]}" ]; then
   echo "ERROR: Left y-axis csv file column undefined." 1>&2
   exit 1
fi

NUM_GRAPHS=$(( ${#CSVCOLUMNSIDX_Y_LEFT[@]} + ${#CSVCOLUMNSIDX_Y_RIGHT[@]} ))
if [ "$NUM_GRAPHS" -gt "${#LINECOLORS[@]}" ]; then
   echo "ERROR: We don't have enough colors for all these graphs. Number of available colors: ${#LINECOLORS[@]}" 1>&2
   exit 1
fi


# Generate Excel chart first, as gnuplot can pause the script
if [ "$DO_EXCEL" -eq 1 ]; then
    echo "Generating Excel chart..."
    generate_excel_chart || exit 1
    echo "Excel chart generated: $OUTXLSXFILE"
fi

# Generate gnuplot chart
if [ "$DO_GNUPLOT" -eq 1 ]; then
   if ! create_op_files; then
      echo "ERROR: Unable to create operation tmp files for gnuplot."
      exit 1
   fi
   
   echo "Generating gnuplot chart..."
   gnuplot_cmd=$(generate_gnuplot_cmd) || exit 1
   echo "$gnuplot_cmd" | gnuplot -p
fi