CoreData : navStatsNotes This page last changed on Aug 01, 2007 by mccann. The columns in the files are:
File | file name
esecs | Start Epoch Seconds (seconds after 1/1/1970 00:00 GMT)
lat | Average Latitude in the file
lon | Average Longitude in the file
maxPres | Maximum pressure in the file
numFreezes | Number of times the system logged the same exact positions (Used to analyze problems the Tiburon USBL system in 2002)
totalDuration | Total duration in minutes of the freezes (Used to analyze problems the Tiburon USBL system in 2002)
recordDuration | Count of records (converted to minutes) when the position was frozen (Used to analyze problems the Tiburon USBL system in 2002)
aveSD | The aveSD column is the average standard deviation between the original and edited nav. If the values are 0.00 then the original and edited data are identical. Larger values will indicate a degraded performance in the nav system - or an incorrectly edited file. There will naturally be a correlation with performance accuracy and depth.
editedNav | Whether an Edited nav file exists or not |
Here is the Perl Script that produces the Statistics reports:
    #!/usr/bin/perl
    #
    # Script to print frozen position statistics for Ventana nav data
    #
    # Mike McCann  25 August 2001  MBARI
    #
    # $Id: navStats.pl,v 1.3 2007/06/06 21:39:16 mccann Exp mccann $
    #
    # $Log: navStats.pl,v $
    # Revision 1.3  2007/06/06 21:39:16  mccann
    # Made work on elvis.  Added comment.
    #
    # Revision 1.2  2002/01/18 17:36:51  mccann
    # Added debug flag and proper checks for tibr data files.
    #
    # Revision 1.1  2002/01/18 17:34:33  mccann
    # Initial revision
    #

    use Statistics::Descriptive;

    my $debug = 0;

    die "\nUsage: $0 <directory where *vnta.txt files are> [#frozen Pos] [excel]\n\n" unless -d $ARGV[0];

    $showOver = $ARGV[1] || 60;     # Show if pos repeats for more times than this
    $outputExcel = 0;
    $outputExcel = 1 if ( $ARGV[2] =~ /excel/i );
    $| = 1;
    $deltaT = 5 / 60;

    #
    # Open directory and get list of all the raw nav files in it. Handle vnta & tibr
    #
    opendir (NAVDIR, $ARGV[0]) || die "Can't open directory $ARGV[0]: $!\n";
    if ( $ARGV[0] =~ /vnta/ ) {
            @navfiles = grep { /vnta.txt$/ } readdir (NAVDIR);
    }
    elsif ( $ARGV[0] =~ /tibr/ ) {
            @navfiles = grep { /tibr.txt$/ } readdir (NAVDIR);
    }
    else {
            die "$0 works only for vnta or tibr directories.\n";
    }
    die "Empty \@navFiles list. Was reading from: $ARGV[0]\n" unless @navfiles;
    closedir (NAVDIR);

    #
    # Print header for output
    # - the aveSD column is the average standard deviation between the original and edited nav
    #   if the values are 0.00 then the original and edited data are identical.  Larger
    #   values will indicate a degraded performance in the nav system.  There will natuarlly
    #   be a correlation with performance accuracy and depth.
    #
    if ( $outputExcel ) {
            ##print "File,esecs,Lat,Lon,maxPres,numFreezes,totalDuration,recordDuration,aveSD,editedData\n";
            print "File\tesecs\tLat\tLon\tmaxPres\tnumFreezes\ttotalDuration\trecordDuration\taveSD\teditedData\n";
    }
    else {
            print "File       str (hhmmss) end  froz Lat    froz lon  sPres    ePres   #   duration\n";
    }

    #
    # Loop through all the files and collect stats on frozen positions
    #
    foreach ( @navfiles ) {
            $file = $_;
            print "file = $file\n" if $debug;
            if ( $outputExcel ) {
            }
            else {
                    print  "$file\n";
            }

            calcFreezes();

            calcSD();

    } # End foreach

    #------------------------------------
    # Subroutines: just to partition the work.  No args passed.  Use global vars
    #

    sub calcFreezes {

            open (NAVFILE, "$ARGV[0]/${file}") || die "Can't open file $ARGV[0]/${file}: $!\n";
            $oldLat = 0;
            $oldLon = 0;
            $sameCount = 0;
            $notSame = 0;
            $prFile = $file;        # Print file string, for listing below
            $prFile =~ s/vnta.txt//;
            $timeSum = 0;
            $recSum = 0;
            $numFrozen = 0;
            $maxPres = 0;
            while ( <NAVFILE> ) {
                    next unless /^\d/;      # Skip rec unless it begins with number

                    #
                    # Read record into my variables
                    #
                    ($Year, $Day, $Time, $Usec, $Lat, $Lon, $East, $North,
                    $Pres, $Head, $Alti, $Pitch, $Roll, $PosFlag, $PresFlag,
                    $HeadFlag, $AltiFlag, $AttitFlag) = split (',', $_);
                    if ( $maxPres < $Pres ) {
                            $maxPres = $Pres;
                    }

                    #
                    # Count records of unchanging Lat & Lon
                    #
                    print "$_\n" if $debug;
                    if ( $Lat == $oldLat && $Lon == $oldLon ) {
                            $sameCount++;
                            if ( $sameCount == 1 ) {
                                    $begUsec = $Usec;
                                    $begTime = $Time;
                                    $begPres = $Pres;
                            }
                    }
                    elsif ( $oldLat != 0 )  {
                            $notSame = 1;
                            $timeDiffSec = $Usec - $begUsec;
                            $duration = sprintf("%6.1f min", $timeDiffSec / 60);
                    } # End if ( $Lat ...

                    #
                    # Print out stats for frozen position after we've counted them
                    #
                    if ( $sameCount > $showOver  &&  $notSame ) {
                            if ( $outputExcel ) {
                            }
                            else {
                                    printf ("%10s %d to %d %f %f %5.1f to %5.1F %3d %s\n",
                                  $prFile, $begTime, $Time, $oldLat, $oldLon, $begPres,
                                  $Pres, $sameCount, $duration);
                                    $prFile = ' ';
                            }
                            $recSum += $deltaT * $sameCount;
                            $sameCount = 0;
                            $notSame = 0;
                            $timeSum += $duration;
                            $numFrozen++;
                            ##die "Debugging\n";;
                    }

                    #
                    # Reset counter if not over our $showOver criteria
                    #
                    elsif ( $sameCount <= $showOver  &&  $notSame ) {
                            $sameCount = 0;
                            $notSame = 0;
                    } # End if ( $sameCount >

                    $oldLat = $Lat;
                    $oldLon = $Lon;

            } # End while (NAVFILE

            #
            # Print summary for this day
            #

            if ( $outputExcel ) {
                    ##print "$prFile,$Usec,$Lat,$Lon,$maxPres,$numFrozen,$timeSum,", sprintf("%6.f", $recSum);
                    print "$prFile\t$Usec\t$Lat\t$Lon\t$maxPres\t$numFrozen\t$timeSum\t", sprintf("%6.f", $recSum);
            }
            else {
                    if ( $numFrozen > 0 ) {
                            printf ("       %3d frozen positions over %3d records long lasting total of %5.1f minutes\n", $numFrozen, $showOver, $timeSum);
                    }
            } # End if ( $outputExcel

    } # End calcFreezes()

    sub calcSD {

            #
            # Create list of lat & lon of differences for the day
            #
            open (NAVFILE, "$ARGV[0]/${file}") || die "Can't open file $ARGV[0]/${file}: $!\n";

            $editedFile = $file;
            $editedFile =~ s/\.txt/edited.txt/;

            if ( -f "$ARGV[0]/${editedFile}" ) {
                    open (EDITEDNAV, "$ARGV[0]/${editedFile}") ||
                            die "Can't open file $ARGV[0]/${editedFile}: $!\n";
            }
            else {
                    print STDERR "Can't open file $ARGV[0]/${editedFile}: $!\n";
                    print STDERR "Skipping this file.\n";
                    if ( $outputExcel ) {
                            ##print ",0,NoEditedNav\n";
                            print "\t0\tNoEditedNav\n";
                    }
                    return;
            }

            undef @lonDiff;
            undef @latDiff;

            while ( ($raw = <NAVFILE>) && ($edited = <EDITEDNAV>) ) {
                    next unless $raw =~ /^\d/;      # Skip rec unless it begins with number

                    #
                    # Read raw record into my variables
                    #
                    my ($Year, $Day, $Time, $Usec, $Lat, $Lon, $East, $North,
                    $Pres, $Head, $Alti, $Pitch, $Roll, $PosFlag, $PresFlag,
                    $HeadFlag, $AltiFlag, $AttitFlag) = split (',', $raw);

                    #
                    # Read edited record into my variables
                    #
                    my (undef, undef, undef, $eUsec, $eLat, $eLon, undef, undef,
                    $ePres, undef, undef, undef, undef, undef, undef,
                    undef, undef, undef) = split (',', $edited);

                    ##print "Lat diff = ", $Lat - $eLat, "  Lon diff = ", $Lon - $eLon, "\n";

                    push @latDiff, $Lat - $eLat;
                    push @lonDiff, $Lon - $eLon;

            } # End while ( <NAVFILE>

            #
            # Compute stats on pos differences
            #
            undef $lonStats;
            undef $latStats;
            my $lonStats = Statistics::Descriptive::Full->new();
            my $latStats = Statistics::Descriptive::Full->new();
            $lonStats->add_data(@lonDiff);
            $latStats->add_data(@latDiff);

            if ( $outputExcel ) {
                    printf "\t %9.6f\n", (
                      sqrt($lonStats->variance) + sqrt($latStats->variance) ) / 2;
            }
            else {
                    printf "       lonDiff SD = %9.6f  latDiff SD = %9.6f,  Ave = %9.6f\n",
                      sqrt($lonStats->variance), sqrt($latStats->variance),
                      (sqrt($lonStats->variance) + sqrt($latStats->variance) ) / 2;
            }

    } # End calcSD()
The Perl script is executed with this shell script:
     #!/bin/sh
    #
    # Run stats on all of our ROV nav data

    outDir="/hosts/tempest/tempbox/mccann/navstats"

    for rov in tibr vnta
    do
            for yrdir in /hosts/tornado/TempNav/200[6-7]
            do
                    dir="$yrdir/$rov"
                    year=`echo $yrdir | cut -d/ -f5`
                    echo navStats.pl $dir 60 excel ">" $outDir/navStats${year}${rov}.txt
                    /bin/rm -f $outDir/navStats${year}${rov}.txt
                    navStats.pl $dir 60 excel > $outDir/navStats${year}${rov}.txt
            done
            #
            # Concatenate all the ROV stat files into one
            #
            cat $outDir/navStats????${rov}.txt > $outDir/navStats${rov}.txt
    done

Document generated by Confluence on Feb 04, 2026 08:22