#!/bin/bash

#########################################
# Name: subcsv
#
# Summary: filter a subset of csv data
#          by time
#
# Description: sorts csv files
# and generates subset using specified
# time window. The csv files must be
# line delimited ASCII, with ISO1806
# timestamps as the primary key, i.e.
# first field uses the format:
# YYYY-MM-DDTHH:MM:SS.sssZ...
#
# Author: k. headley
#
# Copyright 2018 MBARI
#
#########################################

#########################################
# Script configuration defaults
# casual users should not need to change
# anything below this section
#########################################
description="Merge data files, optionally extract specified window"

#################################
# Script variable initialization
#################################
VERBOSE="N"
SORT=${SORT:-"sort"}
GREP=${GREP:-"grep"}
TFILTER=${TFILTER:-"tfilter.sh"}
SORT_OPTS="-k 1,24"

unset DO_HELP
unset DO_TEST
unset DO_INFO
unset ARG_BTIME
unset ARG_ETIME
unset ARG_OFILE
unset IFILES
let "NFILES=0"

#################################
# Function Definitions
#################################

#################################
# name: printUsage
# description: print use message
# args: none
#################################
printUsage(){
    echo
    echo " Description: $description"
    echo
    echo " use: `basename $0` [options] file [file...]"
    echo "  Options:"
    echo "   -b t    : window start"
    echo "   -e t    : window end"
	echo "   -i      : print timestamp summary info"
    echo "   -o path : output file path"
    echo "   -v      : verbose output    [$VERBOSE]"
    echo "   -h      : print use message"
	echo
	echo " Examples:"
	echo
    echo "  # merge and filter four files"
    echo "  subcsv.sh -b 2018-01-01T09:20:00 -e 2018-01-01T09:25:00 -- eee.csv fff.csv ggg.csv hhh.csv"
    echo
    echo "  # print timestamp summary for files and exit"
    echo "  subcsv.sh -i -- \$(ls ./testdata/*20180820*csv)"
	echo
}

########################################
# name: vout
# description: print verbose message to stderr
# args:
#     msg: message
########################################
vout(){
    if [ "${VERBOSE}" == "Y" ] || [ "${VERBOSE}" == "TRUE" ]
    then
    #echo "$1" >&2
    echo "$1"
    fi
}

########################################
# name: imsg
# description: echos with indent
# args:
#     msg: message
#  indent: indent level
########################################
imsg(){
    OIFS=$IFS
    IFS=""
    msg_pad="                                        "
    let "msg_indent=${1}"
    shift
    all="${@}"
    printf "%s%s\n" ${msg_pad:0:${msg_indent}} ${all}
    IFS=$OIFS
}

########################################
# name: msg
# description: echos with default indent
# args:
#     msg: message
########################################
msg(){
	imsg $MSG_INDENT "${*}"
}

########################################
# name: exitError
# description: print use message to stderr
# args:
#     msg:        error message
#     returnCode: exit status to return
########################################
exitError(){
    echo >&2
    echo "`basename $0`: error - $1" >&2
    echo >&2
	exit $2
}

########################################
# name: processCmdLine
# description: do command line processsing
# args:
#     args:       positional paramters
#     returnCode: none
########################################
processCmdLine(){
    OPTIND=1
    #vout "`basename $0` all args[$*]"

    while getopts b:e:hio:tv Option
    do
        #vout "processing $Option[$OPTARG]"
        case $Option in
            b ) ARG_BTIME=$OPTARG
            ;;
            e ) ARG_ETIME=$OPTARG
            ;;
            i ) DO_INFO="Y"
            ;;
            o ) ARG_OFILE=$OPTARG
            ;;
            t ) DO_TEST="Y"
            ;;
            v ) VERBOSE="Y"
            ;;
            h) DO_HELP="Y"
            ;;
            *) exit 0 # getopts outputs error message
            ;;
        esac
    done
}

##########################
# Script main entry point
##########################

###########################
# Argument processing
# Accepts arguments from command line
# or pipe/file redirect

# process command line args
if [ "$#" -eq 0 ]
then
    printUsage
    exit -1
else
    # this ensures that quoted whitespace is preserved by getopts
    # note use of $@, in quotes

    # originally was processCmdLine $*
    processCmdLine "$@"
    let "i=$OPTIND-1"
    while [ "${i}" -gt 0  ]
    do
        shift
        let "i-=1"
    done
fi
IFILES=$*
NFILES=$#

###########################
# Generate options for grep, tfilter

#if [ -z ${ARG_BTIME} ]
#then
#	ARG_BTIME="1970-01-01T00:00:00"
#fi
#OPT_BTIME="-b ${ARG_BTIME}"

if [ ${ARG_BTIME} ]
then
OPT_BTIME="-b ${ARG_BTIME}"
fi

if [ ${ARG_ETIME} ]
then
	OPT_ETIME="-e ${ARG_ETIME}"
fi

if [ ${VERBOSE} == "Y" ]
then
OPT_VERBOSE="-v"
fi

###########################
# debug output
vout ""
vout "$0"
vout "VERBOSE   : [$VERBOSE]"
vout "ARG_OFILE : [$ARG_OFILE]"
vout "ARG_BTIME : [$ARG_BTIME]"
vout "ARG_ETIME : [$ARG_ETIME]"
vout "NFILES    : [$NFILES]"
vout "IFILES    : [$IFILES]"
vout ""

###########################
# Validate options

# try to find tfilter.sh script
# exit if not found
if [ -z $( which ${TFILTER} ) ]
then
	if [ $( find . -name tfilter.sh ) ]
	then
		TFILTER=$( find . -name tfilter.sh )
	fi
fi

if [ -z ${TFILTER} ]
then

	exitError "Could not find tfilter.sh" -1
fi

# try to create output file directory
# if it doesn't exist
if [ ! -z ${ARG_OFILE} ]
then
    odir=$(dirname $ARG_OFILE)
	if [ ! -d ${odir} ]
    then
		mkdir -p $odir
    fi

	if [ ! -d ${odir} ]
	then
		exitError "Could not create directory $odir" -1
	fi
fi

###########################
# Run

if [ ${DO_HELP} ]
then
    printUsage
    exit 0
fi

if [ ${NFILES} -le 0 ]
then
exitError "No input files specified" -1
fi

if [ ! -z ${DO_INFO} ] && [ ${NFILES} -gt 0 ]
then
first=$( ${SORT} ${SORT_OPTS} ${IFILES} | grep -e "^[0-9]" | head -n 1 )
last=$( ${SORT} ${SORT_OPTS} ${IFILES}  | grep -e "^[0-9]" | tail -n 1 )
echo

printf "%25s : %s\n" "first (all files)" ${first:0:23}
printf "%25s : %s\n\n" "last  (all files)" ${last:0:23}

for file in ${IFILES[*]}
do
let "nrec=$(cat ${file} | wc -l)"
first=$( ${SORT} ${SORT_OPTS} ${file} | grep -e "^[0-9]" | head -n 1 )
last=$( ${SORT} ${SORT_OPTS} ${file}  | grep -e "^[0-9]" | tail -n 1 )
printf "%25s [%8d]: %s - %s\n" $( basename ${file} ) ${nrec}  ${first:0:23}  ${last:0:23}
done
echo
exit 0
fi

if [ ! -z ${DO_TEST} ]
then
# Test mode
# print command line and exit
	if [ ${ARG_OFILE} ]
    then
		echo "${SORT} ${SORT_OPTS} ${IFILES} | ${TFILTER} ${OPT_BTIME} ${OPT_ETIME}  ${OPT_VERBOSE} > ${ARG_OFILE}"
    else
		echo "${SORT} ${SORT_OPTS} ${IFILES} | ${TFILTER} ${OPT_BTIME} ${OPT_ETIME}  ${OPT_VERBOSE}"
    fi

else

# Real mode
# - sort the files by timestamp and merge
# - grep to filter out the records before the start of the window
# - use tfilter to extract the records in the window
# - optionally send to file instead of stdout

	if [ ${ARG_OFILE} ]
    then
# output to file
${SORT} ${SORT_OPTS} ${IFILES} | ${TFILTER} ${OPT_BTIME} ${OPT_ETIME}  ${OPT_VERBOSE} > ${ARG_OFILE}
    else
# output to stdout
${SORT} ${SORT_OPTS} ${IFILES} | ${TFILTER} ${OPT_BTIME} ${OPT_ETIME} ${OPT_VERBOSE}
    fi
fi