/** \file
 *
 *  Contains the unserialize "main" code.
 *
 *  Copyright (c) 2007,2008,2009 MBARI
 *  MBARI Proprietary Information.  All Rights Reserved
 *
 */

#include "data/Slate.h"
#include "io/FileInStream.h"
#include "io/FileOutStream.h"
#include "io/ZipEncoder.h"
#include "logger/BinaryLogReader.h"
#include "logger/BinaryLogWriter.h"
#include "logger/DirectoryEntry.h"
#include "logger/DirectoryLogWriter.h"
#include "logger/LogEngine.h"
#include "logger/FileLogReader.h"
#include "logger/FileLogWriter.h"
#include "logger/FilterLogWriter.h"
#include "logger/HasFilename.h"
#include "logger/HDF5LogWriter.h"
#include "logger/JsonLogWriter.h"
#include "logger/KmlLogWriter.h"
#include "logger/LinearApproximationLogWriter.h"
#include "logger/NetCdfLogWriter.h"
#include "logger/NetCdfTableLogWriter.h"
#include "logger/TextLogWriter.h"
#include "logger/TextTableLogWriter.h"
#include "units/Units.h"

#include <cctype>
#include <cstddef>
#include <dirent.h>
#include <math.h>
#include <unistd.h>

bool is_var( const char* arg )
{
    return isalpha( *arg );
}

bool is_child( const FlexArray<char*>& parentList, const char* var )
{
    if( parentList.size() == 0 )
    {
        return true;
    }
    size_t dotAt = Str::Find( var, '.' );
    if( dotAt == Str::NO_POS )
    {
        return false;
    }
    for( unsigned int i = 0; i < parentList.size(); ++i )
    {
        if( 0 == strncmp( var, parentList.get( i ), dotAt ) )
        {
            return true;
        }
    }
    return false;
}

bool contains_chars( FlexArray<char*>* list, char* chars, size_t charsLen )
{
    for( unsigned int i = 0; i < list->size(); ++i )
    {
        if( 0 == strncmp( list->get( i ), chars, charsLen ) )
        {
            return true;
        }
    }
    return false;
}

class Unserialize
{
public:
    static int Run( int argc, char **argv );
};

/**
 *
 *  main() code for the unserialize application.
 *
 *  \ingroup logging
 */
int main( int argc, char **argv )
{
    return Unserialize::Run( argc, argv );
}

int Unserialize::Run( int argc, char **argv )
{
    // Boolean flags corresponding to command line arguments
    bool doBin = false;
    bool doCsv = false;
    bool doDateInName = true;
    bool doDebug = false;
    bool doDir = false;
    bool doDirOnly = false;
    bool doHDF5 = false;
    bool doKml = false;
    bool doKmlSimple = false;
    bool doJson = false;
    bool doJsonFmt = false;
    bool doInterp = false;
    bool doLocalTime = false;
    bool doMatlab = false;
    bool doNetCdfTable = false;
    bool doNetCdf = false;
    bool doAsc = false;
    bool doSyslog = false;
    bool doSymlink = false;
    bool toOutput = false;
    bool toStdOut = false;
    bool inputSet = false;
    bool hideUnits = false;
    bool quiet = false;

    // lists of variables specified for processing
    int attCount = 0;
    int varCount = 0;
    char** attList = NULL;
    char** varList = NULL;
    char** unitList = NULL;

    // names of parent components of variables to include in output
    FlexArray<char*> parentList( false );

    // names of Priority classes to use as aliases for slate file name
    FlexArray<const char*> priorityList( false );

    // arguments for linear approximation decimation
    int approxCount = 0;
    float* approxErrors = NULL;

    // arguments for sub-setting data based on time
    Timestamp startTime( Timestamp::NOT_SET_TIME );
    Timestamp endTime( Timestamp::NOT_SET_TIME );

    // argument for sub-setting based on when a specified mission component is running
    Str whileRunningName;

    // argument for kml output color
    const char* color = NULL;

    // vehicle name to use in KML output
    Str vehicleName( "LRAUV" );

    // syslog severity thresholds for  KML outputs
    Syslog::Severity kmlSimpleSeverity = Syslog::CRITICAL;
    Syslog::Severity kmlSeverity = Syslog::ERROR;

    // syslog severity thresholds for log output
    Syslog::Severity syslogSeverity = Syslog::DEBUG;

    // optional output file name
    Str outputName;

    // input file name(s)
    Str inputFilenames;

    if( argc == 1 )
    {
        printf( "unserialize version %d, can handle slate versions %d to %d\n",
                DirectoryEntry::GetCurrentLogVersion(),
                DirectoryEntry::GetMinimumLogVersion(),
                DirectoryEntry::GetCurrentLogVersion() );
        printf( "Usage: unserialize [options] [slatefile[(\\n|;)slatefile]...] [outputName [approxError] [(unit)]]...\n" );
        printf( "    option -a Slate output written to \"[slateFile].asc\"\n" );
        printf( "    option -b Binary output written to \"[slateFile].bin\"\n" );
        printf( "    option -c CSV table output written to \"[slateFile].csv\"\n" );
        printf( "    option -C kml color: followed by 4 hex bytes indicating alpha, blue, green, and red\n" );
        printf( "    option -d Directory output written to \"[slateFile].dir\"\n" );
        printf( "    option -D Debug reading of input file\n" );
        printf( "    option -e Output until the end time that follows YYYY-MM-DDTHH:MM:SS.SS\n" );
        printf( "    option -h Hide units in text outputs\n" );
        printf( "    option -H HDF5 output written to \"[slateFile].h5\"\n" );
        printf( "    option -i Interpolate tabular outputs (CSV, TSV, NetCDF)\n" );
        printf( "    option -j (for small logs only) JSON output written to \"[slateFile].js\"\n" );
        printf( "    option -J (for small logs only) formatted JSON written to \"[slateFile].js\"\n" );
        printf( "    option -k KML+KMZ output written to \"[slateFile].km(lz)\"\n" );
        printf( "    option -K KML output (2D, minimal syslog, no timestamps) written to \"[slateFile].kml\"\n" );
        printf( "    option -l Syslog output written to \"[slateFile].log\"\n" );
        printf( "    option -L Use local time only -- no GPS correction\n" );
        printf( "    option -m Matlab output written to \"[slateFile].mat\"\n" );
        printf( "    option -n NetCDF v3 single table output written to \"[slateFile].nc\"\n" );
        printf( "    option -N NetCDF v4 output written to \"[slateFile].nc4\n" );
        printf( "    option -o Output to the filename that follows (\"stdout\" is ok)\n" );
        printf( "    option -O Output filename should not have dates appended\n" );
        printf( "    option* -p Output from universals and variables with the parent component that follows\n" );
        printf( "    option* -P Priority alias for \"[slateFile]\" (A[ny], C[ourier], E[xpress], P[riority], or N[ormal])\n" );
        printf( "    option -q Quiet output -- suppress informational messages" );
        printf( "    option -r Output while the MissionComponent that follows is running.\n" );
        printf( "    option -s Output from the start time that follows YYYY-MM-DDTHH:MM:SS.SS\n" );
        printf( "    option -S Syslog severities: x=none,d>=debug,i>=info,e>=err,m>=imp,f>=fault,c=crit\n" );
        printf( "    option -u Units in separate columns in CSV or TSV outputs\n" );
        printf( "    option -v specify vehicle name for KML/KMZ output\n" );
        printf( "    option -y Link file with dates to symlink without dates\n" );
        printf( "    No option: Slate & Syslog outputs generated\n" );
        printf( "    For either -c or -t, must specify slate file and outputNames\n" );
        printf( "    * option may be repeated\n" );
        return -1;
    }

    // Very simple command line parsing.  Note that options may NOT be merged!
    for( int i = 1; i < argc; ++ i )
    {
        char* arg = argv[ i ];
        if( arg[ 0 ] == '-' )
        {
            int argLen = strlen( arg );
            for( int j = 1; j < argLen; ++j )
            {
                switch( arg[ j ] )
                {
                case 'a':
                    doAsc = true;
                    break;
                case 'b':
                    doBin = true;
                    break;
                case 'c':
                    doCsv = true;
                    break;
                case 'C':
                    if( ++i < argc )
                    {
                        color = argv[ i ];
                    }
                    break;
                case 'd':
                    doDir = true;
                    break;
                case 'D':
                    doDebug = true;
                    break;
                case 'e':
                    if( ++i < argc )
                    {
                        endTime = argv[ i ];
                    }
                    break;
                case 'h':
                    hideUnits = true;
                    break;
                case 'H':
                    doHDF5 = true;
                    break;
                case 'i':
                    doInterp = true;
                    break;
                case 'j':
                    doJson = true;
                    break;
                case 'J':
                    doJson = true;
                    doJsonFmt = true;
                    break;
                case 'k':
                    doKml = true;
                    break;
                case 'K':
                    doKmlSimple = true;
                    break;
                case 'l':
                    doSyslog = true;
                    break;
                case 'L':
                    doLocalTime = true;
                    break;
                case 'm':
                    doMatlab = true;
                    break;
                case 'n':
                    doNetCdfTable = true;
                    break;
                case 'N':
                    doNetCdf = true;
                    break;
                case 'o':
                    toOutput = true;
                    if( ++i < argc )
                    {
                        outputName = argv[ i ];
                        if( outputName == "stdout" )
                        {
                            toStdOut = true;
                        }
                    }
                    break;
                case 'O':
                    doDateInName = false;
                    break;
                case 'p':
                    if( ++i < argc )
                    {
                        parentList.push( argv[ i ] );
                    }
                    break;
                case 'P':
                    if( ++i < argc )
                    {
                        char prio = tolower( *argv[ i ] );
                        if( prio == 'a' || prio == 'c' )
                        {
                            priorityList.push( "Courier" );
                        }
                        if( prio == 'a' || prio == 'e' )
                        {
                            priorityList.push( "Express" );
                        }
                        if( prio == 'a' || prio == 'p' )
                        {
                            priorityList.push( "Priority" );
                        }
                        if( prio == 'a' || prio == 'n' )
                        {
                            priorityList.push( "Normal" );
                        }
                        FileLogReader::SetPriorityList( &priorityList );
                    }
                    break;
                case 'q':
                    quiet = true;
                    break;
                case 'r':
                    if( ++i < argc )
                    {
                        whileRunningName = argv[ i ];
                    }
                    break;
                case 's':
                    if( ++i < argc )
                    {
                        startTime = Timestamp( argv[ i ] );
                    }
                    break;
                case 'S':
                    if( ++i < argc )
                    {
                        switch( *argv[ i ] )
                        {
                        case 'x':
                            kmlSeverity = Syslog::NONE;
                            break;
                        case 'd':
                            kmlSeverity = Syslog::DEBUG;
                            break;
                        case 'i':
                            kmlSeverity = Syslog::INFO;
                            break;
                        case 'e':
                            kmlSeverity = Syslog::ERROR;
                            break;
                        case 'm':
                            kmlSeverity = Syslog::IMPORTANT;
                            break;
                        case 'f':
                            kmlSeverity = Syslog::FAULT;
                            break;
                        case 'c':
                            kmlSeverity = Syslog::CRITICAL;
                            break;
                        default:
                            kmlSeverity = Syslog::NONE;
                            break;
                        }
                    }
                    kmlSimpleSeverity = kmlSeverity;
                    syslogSeverity = kmlSeverity;
                    break;
                case 'v':
                    if( ++i < argc )
                    {
                        vehicleName = argv[ i ];
                    }
                    break;
                case 'y':
                    doSymlink = true;
                    break;
                }
            }
        }
        else if( !inputSet )
        {
            inputFilenames = arg;
            inputSet = true;
        }
        else if( is_var( arg ) )
        {
            ++varCount;
        }
    }

    if( Str::EMPTY_STR == outputName &&
            ( Str::NO_POS != inputFilenames.find( ';' ) || Str::NO_POS != inputFilenames.find( '\n' ) ) )
    {
        fprintf( stderr, "Must specify output name with multiple input files.\n" );
        return -2;
    }

    if( 0 < varCount )
    {
        varList = new char*[ varCount ];
        unitList = new char*[ varCount ];
        approxErrors = new float[ varCount ];
        int varIndex = -1;
        for( int i = 1; i < argc; ++i )
        {
            char* arg = argv[ i ];
            if( inputFilenames != arg && outputName != arg )
            {
                if( is_var( arg ) )
                {
                    varList[ ++varIndex ] = arg;
                    unitList[ varIndex ] = NULL;
                    approxErrors[ varIndex ] = nanf( "" );
                }
                else if( varIndex < 0 )
                {
                    continue;
                }
                if( *arg == '(' )
                {
                    unitList[ varIndex ] = arg + 1;
                    for( char* unit = unitList[ varIndex ]; *unit != 0; ++unit )
                    {
                        if( *unit == ')' )
                        {
                            *unit = 0;
                            break;
                        }
                    }
                }
                else
                {
                    approxErrors[ varIndex ] = atof( arg );
                    if( 0.0f == approxErrors[ varIndex ] )
                    {
                        approxErrors[ varIndex ] = nanf( "" );
                    }
                    else
                    {
                        approxCount += 1;
                    }
                }
            }
        }
    }

    // If no output format is specified, output ascii.
    if( !doAsc && !doDir && !doSyslog && !doJson && !doKml && !doKmlSimple && !doHDF5 && !doMatlab && !doNetCdf && !doNetCdfTable && !doBin && !doCsv )
    {
        doAsc = true;
    }

    // Check if we are only generating a directory output...
    if( !doAsc && doDir && !doSyslog && !doJson && !doKml && !doKmlSimple && !doHDF5 && !doMatlab && !doNetCdf && !doNetCdfTable && !doBin && !doCsv )
    {
        doDirOnly = true;
    }

    // Default location for input data...
    if( !inputSet )
    {
        inputFilenames = "Logs/latest/slate";
    }

    // LogEngine to handle syslogs from unserializing
    LogEngine syslogEngine;
    FileOutStream stdOutStream( stdout );
    syslogEngine.addAssociation( new TextLogWriter( stdOutStream ),
                                 new SyslogSeverityRule( quiet ? Syslog::ERROR : Syslog::DEBUG, true ) );

    // Deal with multiple files provided for concatenation...
    int numInputs;
    Str* inputParts;
    if( Str::NO_POS != inputFilenames.find( '\n' ) )
    {
        inputParts = inputFilenames.split( numInputs, "\n" );
    }
    else
    {
        inputParts = inputFilenames.split( numInputs, ";" );
    }
    for( int part = 0; part < numInputs; ++part )
    {
        if( inputParts[part].findLastOf( "0000" ) == inputParts[part].length() - 4 )
        {
            inputParts[part] = inputParts[part].substr( 0, inputParts[part].length() - 4 );
        }
    }

    // If no variables are specified on the command line, the following code
    // interrogates the data or optional directory file to get all the variables
    // and attributes for logging in organized output formats.

    FlexArray<char*>* flexAttList( NULL );
    FlexArray<char*>* flexVarList( NULL );
    FlexArray<char*>* flexUnitList( NULL );
    if( varCount == 0 &&
            ( doCsv || doHDF5 || doJson || doMatlab || doNetCdf || doNetCdfTable ) )
    {
        flexAttList = new FlexArray<char*>( true );
        flexAttList->setDoArrayDelete( true );
        flexVarList = new FlexArray<char*>( true );
        flexVarList->setDoArrayDelete( true );
        flexUnitList = new FlexArray<char*>( true );
        flexUnitList->setDoArrayDelete( true );
        for( int part = 0; part < numInputs; ++part )
        {
            Str& inputFilename = inputParts[part];
            // Read directory entries if they are available.
            // Do this before initializing data loggers.
            Str dirFilename = inputFilename + ".dir";
            if( !doDir )
            {
                Slate::ReadDirectory( dirFilename );
            }

            // Read directory entries if there was no ".dir" file.
            if( !Slate::IsDirectoryRead() )
            {
                Slate::ReadDirectory( inputFilename );
            }

            int daCount = Slate::GetDataAccessCount();
            DataAccess* lastDa;
            DataAccess* da( NULL );
            for( unsigned short daCode = 0; daCode < daCount; ++daCode )
            {
                lastDa = da;
                da = Slate::GetDataAccess( daCode );
                if( NULL == da )  // || da->getAccessorType() != DataAccessor::DATA_WRITER )
                {
                    continue;
                }
                unsigned short elCode = da->getElementCode();
                ElementURI* uri = Slate::GetElementURI( elCode );
                if( NULL == uri || ( uri->find( ':' ) != Str::NO_POS && uri->find( "durationOfLastRun" ) != Str::NO_POS ) )
                {
                    continue;
                }
                int uriLen = uri->length() + 1;
                char* var = new char[uriLen];
                strncpy( var, uri->cStr(), uriLen );
                bool isAttribute = uri->getElementType() == ElementURI::CONFIG_ELEMENT;
                isAttribute |= uri->getElementType() == ElementURI::SETTING_ELEMENT;
                isAttribute |= uri->startsWith( "Config/" );
                isAttribute |= NULL != lastDa
                               && lastDa->getAccessorType() == DataAccessor::DATA_READER
                               && lastDa->getElementCode() == elCode
                               && lastDa->getOwnerCode() == da->getOwnerCode();
                if( isAttribute )
                {
                    if( part == 0 && uri->find( "keyText" ) == Str::NO_POS )
                    {
                        flexAttList->push( var );
                    }
                    continue;
                }

                if( is_child( parentList, var ) && !contains_chars( flexVarList, var, uriLen ) )
                {
                    flexVarList->push( var );
                    const Unit* unit = da->getUnit();
                    if( NULL == unit )
                    {
                        unit = &Units::NONE;
                    }
                    int unitLen( strlen( unit->getName() ) + 1 );
                    char* unitChars = new char[unitLen];
                    strncpy( unitChars, unit->getName(), unitLen );
                    flexUnitList->push( unitChars );
                }
                else
                {
                    delete[] var;
                }
                unsigned short unCode = da->getUniversalCode();
                if( unCode == ElementURI::NO_CODE )
                {
                    continue;
                }
                uri = Slate::GetElementURI( unCode );
                if( NULL == uri )
                {
                    continue;
                }
                uriLen = uri->length() + 1;
                for( unsigned int i = 0; i < flexVarList->size(); ++i )
                {
                    if( strncmp( uri->cStr(), flexVarList->get( i ), uriLen ) == 0 )
                    {
                        continue;
                    }
                }
                var = new char[uriLen];
                strncpy( var, uri->cStr(), uriLen );
                if( !contains_chars( flexVarList, var, uriLen ) )
                {
                    flexVarList->push( var );
                    const Unit* unit = da->getUnit();
                    if( NULL == unit )
                    {
                        unit = &Units::NONE;
                    }
                    int unitLen( strlen( unit->getName() ) + 1 );
                    char* unitChars = new char[unitLen];
                    strncpy( unitChars, unit->getName(), unitLen );
                    flexUnitList->push( unitChars );
                }
                else
                {
                    delete [] var;
                }
            }
            syslogEngine.processQueue( Timespan::ZERO_TIMESPAN );
            if( numInputs > 1 )
            {
                Slate::ClearDirectory();
            }
        }
        attList = flexAttList->getArray();
        attCount = flexAttList->size();
        varList = flexVarList->getArray();
        varCount = flexVarList->size();
        unitList = flexUnitList->getArray();
    }
    else if( numInputs == 1 )
    {
        if( !doDir )
        {
            Str dirFilename = inputFilenames + ".dir";
            Slate::ReadDirectory( dirFilename );
        }
        // Read directory entries if there was no ".dir" file.
        else if( !Slate::IsDirectoryRead() )
        {
            Slate::ReadDirectory( inputFilenames );
        }
    }

    // LogEngine to handle the data coming out of the file
    LogQueue unserializeQueue;
    LogEngine unserializeLogEngine( unserializeQueue );

    // List of LogWriters that will generate files that may need to be renamed
    FlexArray<HasFilename*> filenameWriters( false );

    // Now create LogWriters and associate them with the unserializeLogEngine
    // so they can receive data and write logs later on...
    if( doBin )
    {
        // The next two statements define a writer that can generate binary output, nominally for
        // splitting, sub-setting, decimating, or concatenating logs in the compact binary format.
        // The BinaryLogWriter just generates a binary stream, and leaves file operations up to the FileLogWriter class
        // that takes the BinaryLogWriter as an argument.  More complicated binary formats (i.e., NetCdf) will
        // wrap the file generation into the format log writer.
        BinaryLogWriter* binaryLogWriter( new BinaryLogWriter() );

        // Note that the FileLogWriter is either redirected to stdout, or to a provided name, or
        // to the input name with an extension added
        FileLogWriter* binFileLogWriter( toStdOut ?
                                         new FileLogWriter( stdout, binaryLogWriter ) :
                                         new FileLogWriter( toOutput ? outputName : inputFilenames + ".bin",
                                                 binaryLogWriter ) );
        // Pushing the FileLogWriter onto the filenameWriters list allows  unserialize
        // to do operations on the filename after the file is written, such as renaming to the
        // start and end times of the data contained
        filenameWriters.push( binFileLogWriter );

        // If the command line contains variable names followed by numbers, then those numbers
        // will be put into an array with indices matching the indices of variable names
        // in the varList array, and approxCount will equal the number of variales
        // to be decimated using the LinearApproximationLogWriter
        if( approxCount > 0 )
        {
            // Adding an association basically adds a filter to the data stream being processed.
            // The code below basically says that if any LogEntry other than type SYSLOG_ENTRY is encountered,
            // feed it to the indicated LogWriter, in this case a LinearApproximationLogWriter
            // that performs additional filtering based on entries in the approxErrors list.
            unserializeLogEngine.addAssociation(
                new LinearApproximationLogWriter(
                    binFileLogWriter,
                    varCount, varList, unitList, approxErrors, false, true ),
                new NotRule( new TypeRule( LogEntry::SYSLOG_LOG_ENTRY ) ) );
        }
        // If the command line contains only variable names, just unserialize those
        // variables using a FilterLogWriter than looks for specified variable names.
        // Note that some of the more complex output formats are child classes of
        // FilterLogWriter.
        else if( varCount > 0 )
        {
            unserializeLogEngine.addAssociation(
                new FilterLogWriter(
                    binFileLogWriter,
                    varCount, varList, unitList, false ),
                new NotRule( new TypeRule( LogEntry::SYSLOG_LOG_ENTRY ) ) );
        }
        // If no variables are to be decimated, don't bother wrapping the binFileLogWriter
        // in a LinearApproximationLogWriter, and don't bother filtering.
        else
        {
            unserializeLogEngine.addAssociation(
                binFileLogWriter,
                NULL );
        }
    }
    if( doCsv )
    {
        // The next two statements define a writer that can generate CSV output. Since CSV is just text
        // the TextTableLogWriter just generates text, and leaves file operations up to the FileLogWriter class
        // that takes the TextTableLogWriter as an argument.  More complicated binary formats (i.e., NetCdf) will
        // wrap the file generation into the format log writer.
        TextTableLogWriter* textTableLogWriter(
            new TextTableLogWriter( ",", varCount, varList, unitList, hideUnits, doInterp ) );


        // Note that the FileLogWriter is either redirected to stdout, or to a provided name, or
        // to the input name with an extension added
        FileLogWriter* csvFileLogWriter( toStdOut ?
                                         new FileLogWriter( stdout, textTableLogWriter ) :
                                         new FileLogWriter( toOutput ? outputName : inputFilenames + ".csv",
                                                 textTableLogWriter ) );

        // Pushing the FileLogWriter onto the filenameWriters list allows  unserialize
        // to do operations on the filename after the file is written, such as renaming to the
        // start and end times of the data contained
        filenameWriters.push( csvFileLogWriter );

        // If the command line contains variable names followed by numbers, then those numbers
        // will be put into an array with indices matching the indices of variable names
        // in the varList array, and approxCount will equal the number of variales
        // to be decimated using the LinearApproximationLogWriter
        if( approxCount > 0 )
        {
            // Adding an association basically adds a filter to the data stream being processed.
            // The code below basically says that if a LogEntry of type DATA_LOG_ENTRY or an
            // EventEntry of type START_CYCLE is encountered, feed it to the indicated LogWriter,
            // in this case a LinearApproximationLogWriter that performs additional filtering
            // based on entries in the approxErrors list before feeding .
            unserializeLogEngine.addAssociation(
                new LinearApproximationLogWriter(
                    csvFileLogWriter,
                    varCount, varList, unitList, approxErrors, true ),
                new OrRule( new TypeRule( LogEntry::DATA_LOG_ENTRY ),
                            new EventTypeRule( EventEntry::START_CYCLE ) ) );
        }
        // If no variables are to be decimated, don't bother wrapping the csvFileLogWriter
        // in a LinearApproximationLogWriter
        else
        {
            // If you think it counter-intuitive that the FileLogWriter is provided to
            // the addAssociation method, rather than the TextTableLogWriter, you are not
            // alone.  The architecture would be cleaner if the FileLogWriter was be provided
            // to the TextTableLogWriter's constructor (rather than the other way around),
            // and the TextTableLogWriter provided below.
            unserializeLogEngine.addAssociation(
                csvFileLogWriter,
                new OrRule( new TypeRule( LogEntry::DATA_LOG_ENTRY ),
                            new EventTypeRule( EventEntry::START_CYCLE ) ) );
        }
    }
    if( doDir )
    {
        // Define a writer that can write directory entries to a file.
        DirectoryLogWriter* directoryLogWriter( new DirectoryLogWriter( true ) );

        // Note we do not add to fileLogWriters because we don't want to rename this file

        // Note that there is no option to decimate.

        // Adding an association basically adds a filter to the data stream being processed.
        // The code below basically says that if a LogEntry of type DIRECTORY_LOG_ENTRY is encountered,
        // feed it to the indicated LogWriter.
        unserializeLogEngine.addAssociation(
            toStdOut ?
            new FileLogWriter( stdout, directoryLogWriter ) :
            new FileLogWriter( toOutput ? outputName : inputFilenames + ".dir",
                               directoryLogWriter ),
            new TypeRule( LogEntry::DIRECTORY_LOG_ENTRY ) );
    }
    if( doSyslog )
    {
        // Define a writer that can write ascii to a file.
        // Note that the FileLogWriter is either redirected to stdout, or to a provided name, or
        // to the input name with an extension added
        FileLogWriter* syslogFileLogWriter( toStdOut ?
                                            new FileLogWriter( stdout, new TextLogWriter() ) :
                                            new FileLogWriter( toOutput ? outputName : inputFilenames + ".log",
                                                    new TextLogWriter() ) );

        // Pushing the FileLogWriter onto the filenameWriters list allows  unserialize
        // to do operations on the filename after the file is written, such as renaming to the
        // start and end times of the data contained
        filenameWriters.push( syslogFileLogWriter );

        // Note that there is no currently option to decimate.

        // Adding an association basically adds a filter to the data stream being processed.
        // The code below basically says that if a LogEntry of type SYSLOG_ENTRY is encountered,
        // and the severity of the syslog message equals or exceeds the specified severity
        // feed it to the indicated LogWriter.
        unserializeLogEngine.addAssociation(
            syslogFileLogWriter,
            new SyslogSeverityRule( syslogSeverity ) );
    }
    if( doAsc )
    {
        // Define a writer that can write ascii to a file.
        // Note that the FileLogWriter is either redirected to stdout, or to a provided name, or
        // to the input name with an extension added
        FileLogWriter* ascFileLogWriter( toStdOut ?
                                         new FileLogWriter( stdout, new TextLogWriter() ) :
                                         new FileLogWriter( toOutput ? outputName : inputFilenames + ".asc",
                                                 new TextLogWriter() ) );

        // Pushing the FileLogWriter onto the filenameWriters list allows  unserialize
        // to do operations on the filename after the file is written, such as renaming to the
        // start and end times of the data contained
        filenameWriters.push( ascFileLogWriter );

        // If the command line contains variable names followed by numbers, then those numbers
        // will be put into an array with indices matching the indices of variable names
        // in the varList array, and approxCount will equal the number of variales
        // to be decimated using the LinearApproximationLogWriter
        if( approxCount > 0 )
        {
            unserializeLogEngine.addAssociation(
                new LinearApproximationLogWriter(
                    ascFileLogWriter,
                    varCount, varList, unitList, approxErrors, false ),
                new TypeRule( LogEntry::DATA_LOG_ENTRY ) );
        }
        // If the command line contains only variable names, just unserialize those
        // variables using a FilterLogWriter than looks for specified variable names.
        // Note that some of the more complex output formats are child classes of
        // FilterLogWriter.
        else if( varCount > 0 )
        {
            // Adding an association basically adds a filter to the data stream being processed.
            // The code below basically says that if a LogEntry of type DATA_LOG_ENTRY is encountered,
            // feed it to the indicated LogWriter, in this case a LinearApproximationLogWriter
            // that performs additional filtering based on entries in the approxErrors list before feeding .
            unserializeLogEngine.addAssociation(
                new FilterLogWriter(
                    ascFileLogWriter,
                    varCount, varList, unitList, true ),
                new TypeRule( LogEntry::DATA_LOG_ENTRY ) );
        }
        // If the commandLine does not contain any variable names, output everything!
        // Notice that the Rule is NULL.
        else
        {
            unserializeLogEngine.addAssociation(
                ascFileLogWriter,
                NULL );
        }
    }
    if( doHDF5 || doMatlab )
    {
        // Define a writer that can generate either native HDF5 or Matlab output.

        // Note that the the file is written either to a provided name, or
        // to the input name with an extension added
        HDF5LogWriter* h5mFileLogWriter(
            new HDF5LogWriter( toOutput ? outputName : inputFilenames + ( doMatlab ? ".mat" : ".h5" ),
                               doMatlab, varCount, varList, unitList, attCount, attList ) );

        // Pushing the h5mFileLogWriter onto the filenameWriters list allows  unserialize
        // to do operations on the filename after the file is written, such as renaming to the
        // start and end times of the data contained
        filenameWriters.push( h5mFileLogWriter );

        // If the command line contains variable names followed by numbers, then those numbers
        // will be put into an array with indices matching the indices of variable names
        // in the varList array, and approxCount will equal the number of variales
        // to be decimated using the LinearApproximationLogWriter
        if( approxCount > 0 )
        {
            // Adding an association basically adds a filter to the data stream being processed.
            // The code below basically says that if a LogEntry of type DATA_LOG_ENTRY is encountered,
            // feed it to the indicated LogWriter, in this case a LinearApproximationLogWriter
            // that performs additional filtering based on entries in the approxErrors list before feeding .
            unserializeLogEngine.addAssociation(
                new LinearApproximationLogWriter(
                    h5mFileLogWriter,
                    varCount, varList, unitList, approxErrors, false ),
                new TypeRule( LogEntry::DATA_LOG_ENTRY ) );
        }
        // If no variables are to be decimated, don't bother wrapping the h5mFileLogWriter
        // in a LinearApproximationLogWriter
        else
        {
            unserializeLogEngine.addAssociation(
                h5mFileLogWriter, new TypeRule( LogEntry::DATA_LOG_ENTRY ) );
        }
    }

    if( doJson )
    {
        // The next two statements define a writer that can generate JSON output. Since JSON is just text
        // the JsonLogWriter just generates text, and leaves file operations up to the FileLogWriter class
        // that takes the JsonLogWriter as an argument.  More complicated binary formats (i.e., NetCdf) will
        // wrap the file generation into the format log writer.
        JsonLogWriter* jsonLogWriter(
            new JsonLogWriter( doJsonFmt, varCount, varList, unitList, attCount, attList ) );

        // Note that the FileLogWriter is either redirected to stdout, or to a provided name, or
        // to the input name with an extension added
        FileLogWriter* jsonFileLogWriter( toStdOut ?
                                          new FileLogWriter( stdout, jsonLogWriter ) :
                                          new FileLogWriter( toOutput ? outputName : inputFilenames + ".js",
                                                  jsonLogWriter ) );

        // Pushing the FileLogWriter onto the filenameWriters list allows  unserialize
        // to do operations on the filename after the file is written, such as renaming to the
        // start and end times of the data contained
        filenameWriters.push( jsonFileLogWriter );

        // If the command line contains variable names followed by numbers, then those numbers
        // will be put into an array with indices matching the indices of variable names
        // in the varList array, and approxCount will equal the number of variales
        // to be decimated using the LinearApproximationLogWriter
        if( approxCount > 0 )
        {
            // Adding an association basically adds a filter to the data stream being processed.
            // The code below basically says that if a LogEntry of type DATA_LOG_ENTRY is encountered,
            // feed it to the indicated LogWriter, in this case a LinearApproximationLogWriter
            // that performs additional filtering based on entries in the approxErrors list.
            unserializeLogEngine.addAssociation(
                new LinearApproximationLogWriter(
                    jsonFileLogWriter,
                    varCount, varList, unitList, approxErrors, false ),
                new TypeRule( LogEntry::DATA_LOG_ENTRY ) );
        }
        // If no variables are to be decimated, don't bother wrapping the jsonFileLogWriter
        // in a LinearApproximationLogWriter
        else
        {
            // If you think it counter-intuitive that the FileLogWriter is provided to
            // the addAssociation method, rather than the JsonLogWriter, you are not
            // alone.  The architecture would be cleaner if the FileLogWriter was be provided
            // to the JsonLogWriter's constructor (rather than the other way around),
            // and the JsonLogWriter provided below.
            unserializeLogEngine.addAssociation(
                jsonFileLogWriter,
                new TypeRule( LogEntry::DATA_LOG_ENTRY ) );
        }
    }

    if( doKml || doKmlSimple )
    {
        // The next several statements define a writer that can generate KML or KMZ output. Since KML is just text
        // the KmlLogWriter just generates text, and leaves file operations up to the FileLogWriter class
        // that takes the KmlLogWriter as an argument.  More complicated binary formats (i.e., NetCdf) will
        // wrap the file generation into the format log writer.
        KmlLogWriter* kmlLogWriter = new KmlLogWriter( vehicleName, !doKmlSimple, !doKmlSimple );
        if( NULL != color )
        {
            kmlLogWriter->setColor( color );
        }

        // Note that the FileLogWriter is either redirected to stdout, or to a provided name, or
        // to the input name with the ".kml" extension added. In the case of kmz output,
        // A ZipEncoder is added to the FileLogWriter that further encodes the kml output
        // into a .kmz file once file writing is done.
        FileLogWriter* kmlFileLogWriter;
        if( toStdOut )
        {
            kmlFileLogWriter = new FileLogWriter( stdout, kmlLogWriter );
        }
        else if( doKmlSimple )
        {
            kmlFileLogWriter = new FileLogWriter( toOutput ? outputName : inputFilenames + ".kml",
                                                  kmlLogWriter );
        }
        else
        {
            kmlFileLogWriter = new FileLogWriter( toOutput ? outputName : inputFilenames + ".kml",
                                                  kmlLogWriter, false, new ZipEncoder( ".kmz" ) );
        }

        // Pushing the FileLogWriter onto the filenameWriters list allows  unserialize
        // to do operations on the filename after the file is written, such as renaming to the
        // start and end times of the data contained
        filenameWriters.push( kmlFileLogWriter );

        // The file filtering rules are complicated and differ between the two formats.
        Rule* kmlRule;
        if( doKmlSimple )
        {
            // For the simple format, filter out a lot of "bloat"
            kmlRule = new AndRule( new OrRule( new TypeRule( LogEntry::DATA_LOG_ENTRY ),
                                               new EventTypeRule( EventEntry::START_CYCLE ),
                                               new SyslogSeverityRule( kmlSimpleSeverity ),
                                               new SyslogStartRule( "Started mission" ) ),
                                   new NotRule( new SyslogStartRule( "Started mission Default" ) ),
                                   new NotRule( new SyslogStartRule( "Started mission Startup" ) ),
                                   new NotRule( new SyslogStartRule( "SBIT Failed" ) ),
                                   new NotRule( new SyslogStartRule( "Failed to parse uplink" ) ),
                                   new NotRule( new SyslogStartRule( "Packet size is not" ) )
                                 );
        }
        else
        {
            // For the full format, grab enough data to do minimal compression
            kmlRule = new OrRule( new TypeRule( LogEntry::DATA_LOG_ENTRY ),
                                  new EventTypeRule( EventEntry::START_CYCLE ),
                                  new SyslogSeverityRule( kmlSeverity ) );
        }

        // Note that kmls are either interpolated from very sparse data while
        // they are generated, or they are made more sparse from very dense data.
        if( doInterp )
        {
            // Here a TableLogWriter is used to "expand" sparse data
            unserializeLogEngine.addAssociation(
                new TableLogWriter( kmlFileLogWriter,
                                    3 - ( doKmlSimple ? 1 : 0 ),
                                    KmlLogWriter::GetKmlArgs( doKmlSimple ),
                                    KmlLogWriter::GetKmlUnits( doKmlSimple ),
                                    true ),
                kmlRule );
        }
        else
        {
            // Here a LinearApproximationLogWriter is used to "compress" dense data
            unserializeLogEngine.addAssociation(
                new LinearApproximationLogWriter( kmlFileLogWriter,
                                                  3 - ( doKmlSimple ? 1 : 0 ),
                                                  KmlLogWriter::GetKmlArgs( doKmlSimple ),
                                                  KmlLogWriter::GetKmlUnits( doKmlSimple ),
                                                  KmlLogWriter::GetKmlErrors( doKmlSimple ),
                                                  true ),
                kmlRule );
        }
    }
    if( doNetCdfTable )
    {

        NetCdfTableLogWriter* ncFileLogWriter;

        // If the command line contains variable names followed by numbers, then those numbers
        // will be put into an array with indices matching the indices of variable names
        // in the varList array, and approxCount will equal the number of variables
        // to be decimated using the LinearApproximationLogWriter
        if( approxCount > 0 )
        {
            // Define a writer that can generate NetCDF-version 3 output.
            // Note that the the file is written either to a provided name, or
            // to the input name with an extension added
            ncFileLogWriter = new NetCdfTableLogWriter( toOutput ? outputName : inputFilenames + ".nc",
                    false, varCount, varList, unitList, true );

            // Adding an association basically adds a filter to the data stream being processed.
            // The code below basically says that if a LogEntry of type DATA_LOG_ENTRY or an
            // EventEntry of type START_CYCLE is encountered, feed it to the indicated LogWriter,
            // in this case a LinearApproximationLogWriter that performs additional filtering
            // based on entries in the approxErrors list before feeding .
            unserializeLogEngine.addAssociation(
                new LinearApproximationLogWriter(
                    ncFileLogWriter,
                    varCount, varList, unitList, approxErrors, true ),
                new OrRule( new TypeRule( LogEntry::DATA_LOG_ENTRY ),
                            new EventTypeRule( EventEntry::START_CYCLE ) ) );
        }
        else
        {
            // Define a writer that can generate NetCDF-version 3 output.
            // Note that the the file is written either to a provided name, or
            // to the input name with an extension added
            ncFileLogWriter = new NetCdfTableLogWriter( toOutput ? outputName : inputFilenames + ".nc",
                    false, varCount, varList, unitList, doInterp, attCount, attList );

            // If no variables are to be decimated, don't bother wrapping the ncFileLogWriter
            // in a LinearApproximationLogWriter
            unserializeLogEngine.addAssociation(
                ncFileLogWriter,
                new OrRule( new TypeRule( LogEntry::DATA_LOG_ENTRY ),
                            new EventTypeRule( EventEntry::START_CYCLE ) ) );
        }
        // Pushing the NetCdfTableLogWriter onto the filenameWriters list allows  unserialize
        // to do operations on the filename after the file is written, such as renaming to the
        // start and end times of the data contained
        filenameWriters.push( ncFileLogWriter );
    }
    if( doNetCdf )
    {
        // Define a writer that can generate NetCDF-version 4 output.
        // Note that the the file is written either to a provided name, or
        // to the input name with an extension added
        NetCdfLogWriter* ncFileLogWriter = new NetCdfLogWriter(
            toOutput ? outputName : inputFilenames + ".nc4",
            true, varCount, varList, unitList, attCount, attList );

        // Pushing the NetCdfLogWriter onto the filenameWriters list allows  unserialize
        // to do operations on the filename after the file is written, such as renaming to the
        // start and end times of the data contained
        filenameWriters.push( ncFileLogWriter );

        // If the command line contains variable names followed by numbers, then those numbers
        // will be put into an array with indices matching the indices of variable names
        // in the varList array, and approxCount will equal the number of variables
        // to be decimated using the LinearApproximationLogWriter
        if( approxCount > 0 )
        {
            // Adding an association basically adds a filter to the data stream being processed.
            // The code below basically says that if a LogEntry of type DATA_LOG_ENTRY or an
            // EventEntry of type START_CYCLE is encountered, feed it to the indicated LogWriter,
            // in this case a LinearApproximationLogWriter that performs additional filtering
            // based on entries in the approxErrors list before feeding .
            unserializeLogEngine.addAssociation(
                new LinearApproximationLogWriter(
                    ncFileLogWriter,
                    varCount, varList, unitList, approxErrors, true ),
                new OrRule( new TypeRule( LogEntry::DATA_LOG_ENTRY ),
                            new EventTypeRule( EventEntry::START_CYCLE ) ) );
        }
        else
        {
            // If no variables are to be decimated, don't bother wrapping the ncFileLogWriter
            // in a LinearApproximationLogWriter
            unserializeLogEngine.addAssociation(
                ncFileLogWriter,
                new OrRule( new TypeRule( LogEntry::DATA_LOG_ENTRY ),
                            new EventTypeRule( EventEntry::START_CYCLE ) ) );
        }
    }

    // Keep track of start end end times of data logged, for file renaming later
    Timestamp firstTime( Timestamp::NOT_SET_TIME );
    Timestamp lastTime( Timestamp::NOT_SET_TIME );

    // Handle syslog messages
    Logger logger( "unserialize" );

    // Need to get list of filenames before unserializeLogEngine.uninitialize()
    // is called and associations are deleted.
    FlexArray<Str*> filenames( true );
    if( !toStdOut )
    {
        for( unsigned int i = 0; i < filenameWriters.size(); ++i )
        {
            HasFilename* filenameWriter = filenameWriters.get( i );
            Str oldName = filenameWriter->getFilename();
            if( oldName != Str::EMPTY_STR )
            {
                filenames.push( new Str( oldName ) );
            }
        }
    }

    // timeAdjust is used to correct timestamps when GPS time fixes are obtained.
    Timespan timeAdjust( doLocalTime ? Timespan::INVALID_TIMESPAN : Timespan::ZERO_TIMESPAN );

    // Now we actually read the binary logs are process that data
    for( int part = 0; part < numInputs; ++part )
    {
        Str& inputFilename = inputParts[part];

        // If a "directory" file exists, read it. Otherwise extract directory
        // information from a full pass through the verbose data
        if( !Slate::IsDirectoryRead() )
        {
            // Read directory entries if they are available.
            // Do this before initializing data loggers.
            Str dirFilename = inputFilename + ".dir";
            if( !doDir )
            {
                Slate::ReadDirectory( dirFilename );
            }

            // Read directory entries if there was no ".dir" file.
            if( !Slate::IsDirectoryRead() )
            {
                Slate::ReadDirectory( inputFilename );
            }
        }

        // If we're only generating a directory file, then we are done.
        // Otherwise continue...
        if( !doDirOnly )
        {
            logger.syslog( "Reading data from " + inputFilename, Syslog::INFO );

            BinaryLogReader* binaryLogReader = new BinaryLogReader( logger,
                    startTime, endTime, whileRunningName,
                    &unserializeLogEngine, &syslogEngine, doDebug );
            // Apply GPS time adjustment from previous logs (if any)
            binaryLogReader->setTimeAdjust( timeAdjust );

            // The FileLogReader opens part files and feeds the contents to the
            // binaryLogReadr
            FileLogReader reader( inputFilename, binaryLogReader );

            // Logs are split into parts: handle up to 20 sequential missing parts
            int missedCount = 0;
            while( missedCount < 20 )
            {
                // reader.read() reads an entire part file, or returns 0 if not readable
                missedCount = ( 0 == reader.read() ? missedCount + 1 : 0 );
            }

            // grab timeAdjust so it can be applied to the next set of logs
            timeAdjust = binaryLogReader->getTimeAdjust();

            // Update first and last times for renaming later...
            if( firstTime == Timestamp::NOT_SET_TIME )
            {
                firstTime = binaryLogReader->getFirstTime();
            }
            lastTime = binaryLogReader->getLastTime();
        }

        syslogEngine.processQueue( Timespan::ZERO_TIMESPAN );

        // If we are concatenating several logs, deal with situations
        // where the directory information may change from log to log.
        if( numInputs > 1 )
        {
            unserializeLogEngine.resetInputs();
            Slate::ClearDirectory();
        }

    }

    // Finish writing to log files.
    unserializeLogEngine.uninitialize();

    // Need to change names after unserializeLogEngine.uninitialize()
    // is called and files are closed.
    for( unsigned int i = 0;
            firstTime != Timestamp::NOT_SET_TIME
            && doDateInName && i < filenames.size();
            ++i )
    {
        Str& oldFilename = *filenames.get( i );
        size_t lastDotAt = oldFilename.findLastOf( '.' );
        if( lastDotAt != Str::NO_POS )
        {
            size_t cutAt = lastDotAt;
            Str datePrefix;
            if( Str::EMPTY_STR == outputName &&
                    inputFilenames.find( "slate" ) == inputFilenames.length() - 5 )
            {
                cutAt -= 5;
                datePrefix = Str::EMPTY_STR;
            }
            else
            {
                datePrefix = "_";
            }
            size_t lastSlashAt = oldFilename.findLastOf( '/' );
            Str path( lastSlashAt == Str::NO_POS ? "" : oldFilename.substr( 0, lastSlashAt + 1 ) );
            cutAt -= lastSlashAt == Str::NO_POS ? 0 : lastSlashAt + 1;
            Str clobberPrefix( oldFilename.substr( lastSlashAt == Str::NO_POS ? 0 : lastSlashAt + 1, cutAt )
                               + datePrefix + firstTime.toSmallerString() );
            Str extension( oldFilename.substr( lastDotAt, oldFilename.length() - lastDotAt ) );
            DIR* dir( opendir( path.cStr() ) );
            if( NULL == dir )
            {
                logger.syslog( "Unserialize is unable to open directory: " + path, Syslog::ERROR );
            }
            else
            {
                union // To use thread-safe readdir_r make sure dirent struct is big enough
                {
                    struct dirent rent_;
                    char b_[offsetof( struct dirent, d_name ) + NAME_MAX + 1];
                } di;
                struct dirent* entry;
                while( 0 == readdir_r( dir, &di.rent_, &entry ) && entry != NULL )
                {
                    Str entryName( entry->d_name );
                    if( entryName.startsWith( clobberPrefix )
                            && entryName.endsWith( extension ) )
                    {
                        remove( ( path + entryName ).cStr() );
                    }
                }
                closedir( dir );
            }

            Str newName = clobberPrefix + "_" + lastTime.toSmallerString() + extension;
            Str newFilename = path + newName;

            int err = rename( oldFilename.cStr(), newFilename.cStr() );
            if( err )
            {
                logger.syslog( "Could not rename " + oldFilename + " to " + newFilename + " due to "
                               + strerror( errno ), Syslog::ERROR );
            }
            else
            {
                logger.syslog( "Saved " + newFilename, Syslog::INFO );
                if( doSymlink )
                {
                    if( symlink( newName.cStr(), oldFilename.cStr() ) )
                    {
                        logger.syslog( "Could not link " + newName + " to " + oldFilename + " due to "
                                       + strerror( errno ), Syslog::ERROR );
                    }
                    else
                    {
                        logger.syslog( "Symlink: " + oldFilename, Syslog::INFO );
                    }
                }
            }
            syslogEngine.processQueue( Timespan::ZERO_TIMESPAN );
        }
    }

    // These deletes must follow the unserializeLogEngine.uninitialize() call
    if( NULL != flexAttList )
    {
        delete flexAttList;
    }
    if( NULL != flexVarList )
    {
        delete flexVarList;
    }
    if( NULL != flexUnitList )
    {
        delete flexUnitList;
    }
    else if( NULL != varList )
    {
        delete[] varList;
    }
    if( NULL != approxErrors )
    {
        delete[] approxErrors;
    }
    if( NULL != inputParts )
    {
        delete[] inputParts;
    }

    Slate::ClearDirectory();
    Slate::Uninitialize();

    if( !quiet )
    {
        printf( "Unserialize done.\n" );
    }

    return 0;
}