use strict;
use Switch;
use SSDSSubmit qw('submitSsdsRecord');
use Time::HiRes qw(gettimeofday tv_interval);

use File::Find;
use Time::Local;

=head1 roverParse

roverParse.pl - Parse and submit Rover records

=head2 SYNOPSIS

	perl roverParse.pl {directoryOfFilesToParse}

=head2 DESCRIPTION

The roverParse utilities provide a Perl application to parse Rover data records and submit 
them to SSDS. The module assumes a certain format for the data records and file names. 
The format is the same as the intended format for future Rover records.

=head3 SVN HEADER

C<$Id: roverParse.pl 4101 2009-02-02 04:46:22Z graybeal $>



=head2 Declaration of methods

=cut

our $nDataFilesParsed = 0;
our $nFilesNotParsed = 0;
our $nNotFiles = 0;
our $nDataRecordsSubmittedFile = 0;
our $nDataRecordsSubmittedTotal = 0;
our $nDataBytesSubmitted = 0;
our $nDeploymentsFound = 0;
our $elapsedTime = 0;
our $totalElapsedTime = 0;
our $averageElapsedTime = 0;

our $rawBytesIn=0;
my $nFields=0;

# our $parentId = 1603;  # All the devices have the Rover as their parent
our $parentId = 1003;  # using this for test
my $packetType = 0;
my $packetSubType = 1;  
my $dataDescriptionID = 1;
my $dataDescriptionVersion = 1;
my $timestampSeconds = 0;
my $timestampNanoseconds = 0;
my $sequenceNumber = 0;
my $bufferBytes = q{};
my $bufferLen = length($bufferBytes);

my $nExpectedFields = 0;
my $expectedOptodeFields = 3;  # data fields from device (excludes timestamp)
my $expectedAcmFields = 9;   # data fields from device (excludes timestamp)
my $expectedBatteryFields = 1;   # data fields from device (excludes timestamp)
my $expectedSyslogFields = 10;   # data fields in syslog (excludes timestamp)
my $expectedEventlogFields = 10;   # data fields in eventlog (excludes timestamp)
my $processThisLine = 'true';
my $TIMESTAMP_LAYOUT = '@0 A4 @5 A2 @8 A2 @11 A2 @14 A2 @17 A2 @20 A2' ;  #format to read ISO8601 with hundredths of a second
my $TIMESTAMP_LENGTH = 23 ;
my $CENTI_TO_NANO = 10000000 ;
my $FALSE = q{} ;

my $unknownRecordType = -1;  # should never happen
my $defaultRecordType = 1;
my $eventRecordType = 11;
my $systemRecordType = 21;

my $DATAFILE;

my $_debug = 3;         # print all messages at or below this level (recommended is '1', to get error and warning msgs)
my %_debugLevel = ( ERROR => 0, WARN => 1,  INFO => 2, DEBUG => 3);
my $_testOutputFile = "RoverRecordsToSubmit.txt" ;
my $_noSubmit = q{} ;        # either 'true' to avoid submitting data, or q{} (empty string) for false
my $_maxRecordsPerFile = 5;  # or a big number, like 100000000

our $TESTOUT;



###############################################
# Data Upload Routine for each Rover file 	  #
###############################################

=head 3 doDataUpload

Uses File::Find to call the method with each found file's name.  

The only parameter is the file name, stored in File::Find::name.

Uses a number of variables (still evolving) that are global to this file.

=over 4

=item *
tbd - 

=item *
tbd - 

=cut

sub doDataUpload {
    my $deviceId = q{};
    my $deviceName = q{};
    my $currentFileName = $File::Find::name;
    my $headerLine = q{};
    our $nFields = q{};
    my $logType = q{};
#    my $NO_BUFFER_2 = 0;
#    my $EMPTY_BUFFER = q{};
    my $t0 = 0;
    my @enclosedCommas = q{};
    my $nEnclosedCommas = 0;

    
    $nDataRecordsSubmittedFile = 0;
    
    # First see if it's a file (later on we will also process directories, for their implicit metadata)
    if (-f) { 
        # First print file name
        if ($_debug >= $_debugLevel{'DEBUG'}) {
            print "\tFound file $currentFileName.\n";
        }
        
        if ($currentFileName =~ /(optode|acm|battery|syslog|eventlog)\.csv/) {
                
            open ($DATAFILE, '<', $currentFileName) || die ("** Could not open $currentFileName!\n");
            
            if ($_debug >= $_debugLevel{'INFO'}) {
                print "\t\t\t\tOpened file $currentFileName.\n";
            }
            
            $headerLine = <$DATAFILE> ;
            if ($_debug >= $_debugLevel{'DEBUG'}) {
                print "\t\t\tHeader line has length " . length($headerLine) . ".\n";
            }

            
            # Get the device ID
            my $success = $headerLine =~ /mbariid="(\d+)"/ ;
            if ($success) {
                $deviceId = $1 ;
	            if ($_debug >= $_debugLevel{'INFO'}) {
	                print "\t\t\t\tDevice ID is $deviceId\n";
	            }
	            else {  # error! bad device ID 
	                if ($_debug >= $_debugLevel{'ERROR'}) {
	                    print "Device ID could not be obtained. Searched string was $headerLine.\n" .
	                        "File will not be ingested.\n\n" ;
	                }
	                next;  # might want a nicer way to exit routine
	            }
            }

            
            # Initialize sequence number for this file (we'll restart the sequence number every file)
            $sequenceNumber = 0;
            
            # Set the number of expected fields  and the log type according to the file type.
            switch  ($currentFileName) {
                case /optode\.csv/  { 
                    $nExpectedFields = $expectedOptodeFields; 
                    $logType = 'Data';
                    $packetSubType = $defaultRecordType;
                }
                case /acm\.csv/     { 
                    $nExpectedFields = $expectedAcmFields;                     
                    $logType = 'Data';
                    $packetSubType = $defaultRecordType;
                }
                case /battery\.csv/  { 
                    $nExpectedFields = $expectedBatteryFields; 
                    $logType = 'Data';
                    $packetSubType = $defaultRecordType;
                }
                case /syslog\.csv/   {
                    $nExpectedFields = $expectedSyslogFields; 
                    $logType = 'System';
                    $packetSubType = $systemRecordType;
                }
                case /eventlog\.csv/   {
                    $nExpectedFields = $expectedEventlogFields;  
                    $logType = 'Event';
                    $packetSubType = $eventRecordType;
                }
                other { 
                    $nExpectedFields = 0; 
                    $logType = 'unknown';
                    $packetSubType = $unknownRecordType;
                }
            }
            
            if ($_debug >= $_debugLevel{'DEBUG'}) {
                print "\t\t\tExpecting $nExpectedFields fields of data.\n";
            }
            while (($nDataRecordsSubmittedFile < $_maxRecordsPerFile) && (my $dataRecord = <$DATAFILE>)) {
                
                # get the timestamp and data buffer
                our ($timestamp, $dataBuffer) = $dataRecord =~ /^(.*?),(.*)$/ ;
                
                if (length($timestamp) != $TIMESTAMP_LENGTH) {   # probably bad timestamp
                
	                if ($_debug >= $_debugLevel{'ERROR'}) {
	                    print "Bad timestamp length (" . length($timestamp) . " bytes). \n\tFile=$currentFileName " .
	                        "Line number=$.  \n\tTimestamp read=$timestamp<< Buffer: $dataBuffer\n" .
	                        "This record will not be submitted.\n\n" ;
	                }
	            }
                else { # probably good timestamp, process record
	               
	                # $timestampSeconds = str2time($timestamp) ;
	                my ($year, $month, $day, $hour, $minute, $second, $hundredths) = unpack $TIMESTAMP_LAYOUT, $timestamp ;
	                $timestampSeconds = timegm($second, $minute, $hour, $day, $month - 1, $year);
	                $timestampNanoseconds = $hundredths * $CENTI_TO_NANO ;
	                
	                if ($_debug >= $_debugLevel{'DEBUG'}) {
	                    print "\t\t\t\tTimestamp string, epoch seconds: $timestamp, $timestampSeconds.$timestampNanoseconds\n";
	                }
	            
	            
	                $bufferLen = length($dataBuffer);
	                
	                my @fields = split(/,/, $dataBuffer);
	                $nFields = scalar(@fields);

	                if ($_debug >= $_debugLevel{'DEBUG'}) {
	                    print "\t\t\t\tNumber of fields in data buffer: $nFields\n " ;
	                }
	                
	                if ($bufferLen == 0 && $_debug >= $_debugLevel{'WARN'}) {
	                    print "Warning! Empty data buffer (submitting anyway).\n" ;
	                }
	                   
	                # there should be a certain number of fields including timestamp
	                if (($nExpectedFields > 0) && ($_debug >= $_debugLevel{'WARN'}) ) {
	                    # see if there are extra commas within quotes
	                    
	                    @enclosedCommas = ($dataBuffer =~ /"(.*,.*)+"/g);
	                    $nEnclosedCommas = scalar(@enclosedCommas);
	                    if ( $_debug >= $_debugLevel{'DEBUG'} ) {
	                        print "\t\t\t\tNumber of enclosed commas found: $nEnclosedCommas (@enclosedCommas)\n";
	                    }
	                    
	                    $nFields = $nFields - $nEnclosedCommas;
	                    if ($nFields != $nExpectedFields) { 
		                    print "Warning! Wrong number of data fields -- " .
		                        "expected $nExpectedFields but found $nFields data fields " .
		                        "(submitting anyway).\n" ;
		                }
	                }
	                
	                # System logs have some multi-line records (e.g., from modem error outputs)
	                # This processing puts all the pieces of the line into a single record
	                
	                $processThisLine = 'true';
	                
	                if (($logType eq 'System') && ! ( $dataBuffer =~ /["]$/) ) {
	                            
                       	if ($_debug >= $_debugLevel{'INFO'}) {
       					     print "\t\tMulti-line record found at line $.:\n$dataBuffer\n" ;
   					    }
                    	                            
                        # read new lines until we find the end of the message
                        until (( $dataBuffer =~ /["]$/) || (eof)) {
                            $dataBuffer .= <$DATAFILE> ;
                            s/\r// ; # eliminate any extra carriage return (note some in-message CRs may get deleted)
                            $bufferLen = length($dataBuffer);
                            
	                       	if ($_debug >= $_debugLevel{'DEBUG'}) {
	       					     print "\t\t\tLine $. appended to create:\n$dataBuffer\n" ;
	   					    }
                            
                        }  ;
                        
                        if (eof) {
                        
                        	if ($_debug >= $_debugLevel{'ERROR'}) {
       					        print "*** ERROR! *** Abrupt end of line found at line $. of $currentFileName:\n$_\n" ;
   					        }
       					    $processThisLine = $FALSE;

                       }
                       else {
                       	   if ($_debug >= $_debugLevel{'INFO'}) {
       					     print "\t\tMulti-line record maintained from lines through $.:\n$dataBuffer\n" ;
   					       }
   					       
   					   }
   					    
                    } # end check for multiline message
	                
	                $sequenceNumber++;
	                $bufferBytes = $dataBuffer;

	                if ( $processThisLine ) {
		                if ($_noSubmit) {
		                    print $TESTOUT "AT $timestamp DEVICE $deviceId SUBMIT $bufferLen RAW BYTES : \n\t$dataBuffer\n" ;
		                    $rawBytesIn = $bufferLen;
		                    if ($_debug >= $_debugLevel{'DEBUG'} ) {
		                        print "\t\t\tRecord fields: " . 
		                            join("::",$deviceId,$parentId,$packetType, $packetSubType, $dataDescriptionID, 
		                            $dataDescriptionVersion,$timestampSeconds, $timestampNanoseconds, $sequenceNumber,
		                            $bufferLen, $bufferBytes) . "\n" ;
		                    }
		                    
		                }
		                else {  # submit the data for real
	                        $t0 = [gettimeofday];
		                    $rawBytesIn = SSDSSubmit::submitSsdsRecord ($deviceId,$parentId,
		                    $packetType, $packetSubType, $dataDescriptionID, $dataDescriptionVersion,
		                    $timestampSeconds, $timestampNanoseconds, $sequenceNumber,$bufferLen, $bufferBytes);
		                    $elapsedTime = tv_interval ( $t0 );
		                    $totalElapsedTime += $elapsedTime;
		                    if ($_debug >= $_debugLevel{'DEBUG'} ) {
		                        print "\t\t\t\tElapsed time (Record, Total): $elapsedTime, $totalElapsedTime\n" ;
		                    } 
		                }
		                
		                if ($_debug >= $_debugLevel{'INFO'}) {
		                    print "\t\tDevice $deviceId submitting $rawBytesIn bytes (file line $.).";
		                }
		                
		                $nDataRecordsSubmittedFile++;
		                $nDataRecordsSubmittedTotal++;
		                $nDataBytesSubmitted += $rawBytesIn;
		                
		                if ($_debug >= $_debugLevel{'DEBUG'}) {
		                    print "\n\t\t\t\tRecords (File, Total), Bytes Submitted: " .
		                    "$nDataRecordsSubmittedFile, $nDataRecordsSubmittedTotal, $nDataBytesSubmitted\n";
		                }
	                }
	                else {  # don't process this line, but set the flag back to true for the next line
	                    $processThisLine = 'true';
	                }
	            }
	            
            } # end while stepping through original file
            
            close ($DATAFILE);
            
            if ($_debug >= $_debugLevel{'INFO'}) {
                print "\t\tParsed $currentFileName \n" ;
                print "\t\tSubmitted $nDataRecordsSubmittedFile data records from this file " .
                "($nDataRecordsSubmittedTotal data records total).\n" ;
            }
            $nDataFilesParsed++;
            
        } else {
        
           if ($_debug >= $_debugLevel{'DEBUG'}) {
                print "\t\t\tNot a file I parse: $currentFileName\n" ;
            }
            $nFilesNotParsed++;
        }
        
    } else {         
    
        if ($_debug >= $_debugLevel{'DEBUG'}) {
            print "\t\t\tNot a file: $currentFileName\n" ;
        }
        $nNotFiles++;
        
    }   # only execute subroutine if this is a file
            
} # end of subroutine


    
if ($_debug >= $_debugLevel{'INFO'}) {
	print "Starting process at " . localtime() . ".\n\n" ;
}

@ARGV = ("/Users/graybeal/Desktop/RoverConversionData") unless @ARGV;

if ($_noSubmit) {
	print "Using test output file $_testOutputFile.\n\n" ;
	 open ($TESTOUT, ">", $_testOutputFile) || die ("** Could not open $_testOutputFile!\n");
	 print $TESTOUT "Starting test " . localtime() . "\n\n" ;
}

find (\&doDataUpload, @ARGV);

print "Done!\n\tData files parsed: $nDataFilesParsed\n\tFiles not parsed: $nFilesNotParsed\n\tItems not files: $nNotFiles\n" ;
print "Total data records submitted: $nDataRecordsSubmittedTotal.\n";
$averageElapsedTime = $totalElapsedTime / $nDataRecordsSubmittedTotal;
if ($_debug >= $_debugLevel{'INFO'}) {
    print "Elapsed time for submissions: $totalElapsedTime seconds. Per record: $averageElapsedTime.\n";
}

if ($_noSubmit) {
    print $TESTOUT "\n\nFinishing test at " . localtime() . "\n" ;
    close $TESTOUT || die ("** Could not close $_testOutputFile!\n");
}

if ($_debug >= $_debugLevel{'INFO'}) {
	print "Finished process at " . localtime() . ".\n\n" ;
}