#!/usr/bin/perl 
use strict;
use warnings;
use Switch;

use File::Find;

=head1 formatDataFiles

formatDataFiles.pl - Format pre-existing Rover data files for submission
 
C<$Id: formatDataFiles.pl 4597 2009-04-23 16:13:34Z graybeal $>


=head1 SYNOPSIS

perl formatDataFiles  

=head1 DESCRIPTION

The formatDataFiles utility is a one-off, written to convert the existing Rover data files (pre-2009) to the new format. It may have useful code for one thing or another, but is not expected to be reused on the Rover project once these files are in SSDS.

The utility assumes the original raw files have already been reorganized into the structure future Rover missions will produce (although it will actually work on any individual data file or set of them).

It uses the name of the file, the dates on records within the file, and the path name of the file, as metadata to choose how to process records in the file.

If you want to run or test it on a particular file, copy that file and its contextual path to another directory (typically the one specified as the default

This script leaves the files in a partially processed state -- the old files are left in place. To 
complete or undo the transition, edit undoFormatFiles.pl accordingly and then run it.

=cut

our $nDataFilesModified = 0;
our $nFilesNotModified = 0;
our $nNotFiles = 0;
our $defaultDirectory = "/Users/graybeal/Desktop/RoverConversionData";

my %_debugLevel = ( ERROR => 0, WARN => 1,  INFO => 2, DEBUG => 3);
my $_debug = 2;         # print all messages at or below this level 
                        #   (recommended is '1' or higher, to get error and warning msgs)

my $FALSE = q{};



sub do_substitutions{
    my $origSuffix = '_orig';
    my $suffixLen = length ($origSuffix);
    my $tempSuffix = '_temp';
    my $deviceDevPort = q{};
    my $deviceId = q{};
    my $deviceName = q{};
    my $currentFileName = $File::Find::name;
    our $headerLine = q{};
    my $thisRecord = q{}; 
    
    my $printThisLine = 'true';  
    my $logType = q{};

    
    # substitute strings here
    # First print file name
    if (-f) { 
    	if ($_debug >= $_debugLevel{'INFO'}) {
            print "\tFound file $currentFileName.\n";
        }

        print "\tFound file $currentFileName.\n";
        
        if ($currentFileName =~ /(optode|acm|battery|syslog|eventlog)\.csv/) {
        
            open (OLD, "<", $currentFileName);
            open (NEW, ">", $currentFileName . $tempSuffix);
            
            while (<OLD>) {
                # make the changes
                switch  ($currentFileName) {
                    case /optode\.csv/  {

                        if ($. == 1) {
                            switch ($currentFileName) {
                                case /starboard_optode/ {
                                    $deviceDevPort = "/dev/starboard_optode";
                                    $deviceId = "1632" ;
                                    $deviceName = "Rover Starboard Optode" ;
                                }
                                case /port_optode/ {
                                    $deviceDevPort = "/dev/port_optode";
                                    if ($currentFileName =~ /-2007\.06/) {
                                        $deviceId = "1700"; # unknown device
                                    }
                                    else {
                                        $deviceId = "1630" ;
                                    }
                                    $deviceName = "Rover Port Optode" ;
                                }
                                
                                case /ref_optode/ {
                                    $deviceDevPort = "/dev/ref_optode";
                                    if ($currentFileName =~ /-2007\.11/) {
                                        $deviceId = "1700"; # unknown device
                                    }
                                    else {
                                        $deviceId = "1650" ;
                                    }
                                    $deviceName = "Rover Reference Optode" ;
                                }
                                
                            }  # end of location selection switch
                                
                            # Put first line together
                            $_ = "DateTime (ISO8601), Oxygen (uM), " .
                                  "Oxygen Saturation (%), Water Temperature (degrees C), " .
                                  "<Device port=\"$deviceDevPort\" mbariid=\"$deviceId\" model=\"$deviceName\" />\n" ;
                        
                        } 
                        else {    # handle rest of the lines
                        
	                       	if ($_debug >= $_debugLevel{'DEBUG'}) {
	       					     print "\tLine $. before:>>$_<<\n";
	   					    }
	   					    
                           if (/^200/) {  # only process and print the lines that are reasonable data lines
                           
	                           s/\r// ; # eliminate any extra carriage return from Windows handling
	                            
		                        
	                           if (/^200706/) {
	                        	   s/^(200[^Z]+?) ?, ?/$1Z,/  ;  # adding 'Z' to timestamp, removing optional blanks (no mission time to remove)
	                           }
	                           else {
	                               s/^(200[^Z]+?),.*?, ?/$1Z,/  ;  # adding 'Z' to timestamp, removing mission time (and optional blank after comma)
	                           }
		                       s/^(\d\d\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)/$1-$2-$3T$4:$5:$6/ ; # turning original timestamp format into ISO 8601
	                        
	                        }    
                            else { # not good data
	                    
		                    	$printThisLine = $FALSE;
			                    if ($_debug >= $_debugLevel{'WARN'}) {
		       					     print "\tWarning! Line $. of file " . $currentFileName . " incorrectly formatted, not processed: \n$_\n";
		   					    }
		                    
		                    } # end check for good data


	                       	if ($_debug >= $_debugLevel{'DEBUG'}) {
	       					     print "\tLine $. after :>>$_<<\n";
	   					    }
                        }   # end of test on line number

                        
                    }  # end of optode case
                    
                    case /acm\.csv/ {
                    
                        if ($. == 1) {
                            # ACM *usually* doesn't have first header line, so insert this one before first line
                            $deviceDevPort = "/dev/acm";
                            $deviceId = "1637" ;
                            $deviceName = "Rover Currents Sensor" ;

                            # Put first line together
                            $headerLine = "DateTime (ISO8601)Z,ACM Time (hh_mm_ss), ACM Date (mm_dd_yyyy), " .
                                  "Heading (degrees), Battery Voltage (volts), " .
                                  "ACM Tilt Y: Roll (degrees), ACM Tilt Y: Pitch (degrees), " .
                                  "North Current (degrees), East Current (degrees), Temperature (degrees C), " .
                                  "<Device port=\"$deviceDevPort\" mbariid=\"$deviceId\" model=\"$deviceName\" />\n" ;
                            
                            print NEW "$headerLine" ;
                            
                            if (/Rover Timestamp/) {  # the first line IS a header line, read past it
                                $_ = <OLD> ;
                            }
                        }
                                                
                        # ACM didn't have first header line, so execute the nominal else clause even for first line
                        #} else {    # handle rest of the lines
                        
                        s/\r// ; # eliminate any extra carriage return

                       	if ($_debug >= $_debugLevel{'DEBUG'}) {
       					     print "\tLine $. before:>>$_<<\n";
   					    }
                        
                        if (/^200/) {  # only process and print the lines that are reasonable data lines
                         
                            if (/^20070608/) {  # the first 2 ACM data files do not have mission time, so check for that
                                s/^(200[^Z]+?) ?, ?/$1Z,/  ;  # add 'Z' to timestamp, remove optional space after comma
                            }
                            else {
	                            s/^(200[^Z]+?) ?,.*?, ?/$1Z,/  ;  # add 'Z' to timestamp, removing Mission Time (and optional space after comma)
	                        }
	                        s/^(\d\d\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)/$1-$2-$3T$4:$5:$6/ ; # turning original timestamp format into ISO 8601
	                        s/,[0-9.]+?$//  ;  # remove last item (which was calculated)
	                        
	                        if (/^2008-(09|10|11)-/) { # for 2008-10/2008-11 missions, remove additional last item (also calculated)
	                            s/,[0-9.]+?$//  ; 
                            }
	                            
                        }
	                    else { # not good data
	                    
	                    	$printThisLine = $FALSE;
		                    if ($_debug >= $_debugLevel{'WARN'}) {
	       					     print "\tWarning! Line $. of file $currentFileName incorrectly formatted, not processed: \n$_\n";
	   					    }
	   					}
                            
                        #}   # end of test on line number
                                            
                    }  # end of acm case
                    
                    case /battery\.csv/ {
                    
                        if ($. == 1) {
                            # battery often doesn't have first header line, so insert this one before first line
                            $deviceDevPort = "/dev/battery";
                            $deviceId = "1657" ;
                            $deviceName = "Rover Battery Backup/Power Supply" ;

                            # Put first line together
                            $headerLine = "DateTime (ISO8601)Z, Rover Battery Voltage (volts), " .
                                  "<Device port=\"$deviceDevPort\" mbariid=\"$deviceId\" model=\"$deviceName\" />\n" ;
                            
                            print NEW "$headerLine" ;
                        }
                                                
                        # battery may or may not have first header line, so execute the nominal else clause even for first line
                        #} else {    # handle rest of the lines
                        
                        if (/^200/) {  # only process and print the lines that are reasonable data lines
                        
	                        s/\r// ; # eliminate any extra carriage return

	                       	if ($_debug >= $_debugLevel{'DEBUG'}) {
	       					     print "\tLine $. before:>>$_<<\n";
	   					    }
	                        
	                        s/^(200[^Z]+?),.*?, ?/$1Z,/  ;  # adding 'Z' to timestamp, removing Mission Time (and possible blank after comma)
	                        s/^(\d\d\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)/$1-$2-$3T$4:$5:$6/ ; # turning original timestamp format into ISO 8601
	                        
	                    } 
	                    else { # not good data
	                    
	                    	$printThisLine = $FALSE;
		                    if ($_debug >= $_debugLevel{'WARN'}) {
	       					     print "\tWarning! Line $. of file $currentFileName incorrectly formatted, not processed: \n$_\n";
	   					    }

	                    
	                    } # end check for good data
	                        
                            
                        #}   # end of test on line number
                                            
                    }  # end of battery case
                    
                    case /(syslog|eventlog)\.csv/ {
                    
                        if ($currentFileName =~ /syslog/) {
                            $logType = 'System';
                        }
                        else {
                            $logType = 'Event';
                        }
                    
                        if ($. == 1) {
                            # system/event log often doesn't have first header line, so insert this one before first line
                            $deviceDevPort = "sw-system-$logType";
                            $deviceId = "1633" ;
                            $deviceName = "Rover CPU ($logType Process)" ;

                            # Put first line together
                            $headerLine = "DateTime (ISO8601)Z, Mission Time (seconds), " .
                                  "ModuleGroup1, ModuleGroup2 (), ModuleGroup3 (), ModuleGroup4 (), ModuleGroup5 ()," .
                                  "ModuleGroup6, ModuleGroup7 (), ModuleGroup8 (), $logType Log Message (), " .
                                  "<Device port=\"$deviceDevPort\" mbariid=\"$deviceId\" model=\"$deviceName\" />\n" ;
                            
                            print NEW "$headerLine" ;
                        }
                                                
                        # log may or may not have first header line, so execute the nominal else clause even for first line
                        #} else {    # handle rest of the lines
                                               
                        if (/^200/) {  # only process and print the lines that are reasonable data lines
                        
	                        s/\r// ; # eliminate any extra carriage return

	                       	if ($_debug >= $_debugLevel{'DEBUG'}) {
	       					     print "\tLine $. before:>>$_<<\n" ;
	   					    }
	                        
	                        s/^(200[^Z]+?),/$1Z,/  ;  # adding 'Z' to timestamp
	                        
	                        # check for multiline message in syslog (all good log messages will end in "\r)
	                        if (($logType eq 'System') && ! ( /["]$/) ) {
	                            
		                       	if ($_debug >= $_debugLevel{'INFO'}) {
		       					     print "\t\tMulti-line record found at line $.:\n$_\n" ;
		   					    }
	                        	                            
	                            # read new lines until we find the end of the message
	                            until ((/["]$/) || (eof)) {
	                                $_ .= <OLD> ;
	                                s/\r// ; # eliminate any extra carriage return (note some in-message CRs may get deleted)
	                                
			                       	if ($_debug >= $_debugLevel{'DEBUG'}) {
			       					     print "\t\t\tLine $. appended to create:\n$_\n" ;
			   					    }
	                                
	                            }  ;
	                            
	                            if (eof) {
	                            
	                            	if ($_debug >= $_debugLevel{'ERROR'}) {
		       					        print "*** ERROR! *** Abrupt end of line found at line $. of $currentFileName:\n$_\n" ;
		   					        }
		       					    $printThisLine = $FALSE;
   
	                           }
	                           else {
		                       	   if ($_debug >= $_debugLevel{'INFO'}) {
		       					     print "\t\tMulti-line record maintained from lines through $.:\n$_\n" ;
		   					       }
		   					       
		   					   }
		   					    
	                        } # end check for multiline message
	                        
	                    } 
	                    else { # not good data
	                    
	                    	$printThisLine = $FALSE;
	                    
	                    } # end check for good data
	                        
                            
                        #}   # end of test on line number
                                            
                    }  # end of system/event log case
                    
                
                }  # end of switch
                
#                # print out each new line if warranted
                if ($printThisLine) {
                    print NEW "$_"  ;
                }
                else {
                    $printThisLine = 'true'; 
                }
                    

            } # end while stepping through original file
            
            close (OLD);
            close (NEW);
            
            rename ($currentFileName, $currentFileName . $origSuffix );
            print "\t\tSaved modified file as $currentFileName" . "$origSuffix \n" ;
            $nDataFilesModified++;
            
        } else {
        
            print "\tNot a file I modify: $currentFileName\n" ;
            $nFilesNotModified++;
        }
        
        
    } 
    else {         
    
        print "\tNot a file: $currentFileName\n" ;
        $nNotFiles++;
        
    }   # only execute subroutine if this is a file
            
} # end of subroutine
    
@ARGV = ($defaultDirectory) unless @ARGV;

find (\&do_substitutions, @ARGV);

print "Done!\n\tData files modified: $nDataFilesModified\n\tFiles not modified: $nFilesNotModified\n\tItems not files: $nNotFiles\n" ;