10912280 80 10 80 2 10945035 11013509 6 2010-01-04 14:31:35.077 2010-01-04 16:17:35.783 10912281 10912282 10912286 10912288 10912290 8356145 80 10 80 2 8388898 1 2009-01-13 13:26:51.170 2009-01-13 13:26:51.170 1179945 80 10 80 2 1212710 10455990 10455991 1343552 1933368 10 2007-08-03 07:17:05.643 2009-06-16 10:34:34.160 1179946 1179947 2162927 7766118 7766120 7766122 7766124 9372424 10354987 32809 theme.key ]]> 32810 SSDS false custom ]]> 6259415 2 6226616 1 2008-10-21 08:57:57.783 2008-10-21 08:57:57.787 4456506 80 10 80 2 4489270 4555174 4555175 4555176 2 2008-05-30 15:28:37.337 2008-05-30 15:57:40.887 4456509 13926583 80 10 80 2 13959345 5 2010-07-27 14:50:24.337 2010-08-02 06:58:42.843 13926586 13926627 13926661 13926683 32818 2]]> 2 9 10 45 46 47 48 49 50 51 52 53 54 55 56 57 58 4882506 4882507 19300614 2006-10-17 21:39:33.590 2008-05-30 10:50:44.757 global 10356413 8355863 10 80 8355863 2 10389125 10461968 10461969 10461970 10461971 10461972 10461973 8 2009-10-07 10:47:14.877 2009-10-07 22:55:32.233 10356415 10356417 10356420 10356436 10356446 10356453 10356455 10912252 596 10 596 2 10945007 11013138 1 2009-12-28 13:54:14.490 2009-12-28 13:54:14.490 11240414 841 10 80 841 2 11273151 13205730 13205731 13205732 13205733 13205734 11 2010-03-10 21:41:49.343 2010-06-03 17:11:55.297 11240415 11240417 11240419 11240421 11240422 11240424 11240426 13140041 13140043 13140045 10355890 80 10 80 2 10388616 10458414 8 2009-08-11 16:05:27.267 2009-08-11 16:52:13.317 10355891 10355894 10355895 10355897 10355898 10355900 10355902 10485949 10485950 8356090 80 10 80 2 8388846 4 2009-01-12 13:08:50.467 2010-06-08 10:35:51.147 10912291 10912293 13140051 10912345 80 10 80 2 10945092 1 2010-01-05 11:43:47.797 2010-01-05 11:43:47.797 11797071 80 11797073 10 80 2 11829824 17564847 13 2010-04-27 15:25:51.250 2011-06-09 08:02:11.717 16777248 16777250 16777251 16777253 16777255 16777257 16777259 17465499 17465504 17465506 17465507 17465509 11797073 11797071 10 80 11797071 2 11829826 3 2010-04-27 15:32:49.873 2010-04-27 15:35:39.053 11797075 11797077 841 80 11240414 10 80 2 838 11341540 11341541 11341542 11341543 11341544 11341545 11341546 11341547 11341548 11341549 1343543 40 2007-02-08 12:50:50.907 2010-03-10 21:19:01.183 844 847 849 850 851 4456469 4456477 4456478 5832936 5832939 5832941 8913006 8913010 8913012 8913014 8913016 8913017 8913019 8913027 8913029 8913100 10355400 10355406 10355411 10355413 10355415 10355416 10355417 10355419 10355421 10355423 10355426 10355429 10355431 10355433 10355436 10355438 10355440 11240412 10355664 10 10 2 10388392 25559138 25559139 25559140 10322082 10322092 10322148 10322158 10322168 10322170 10322228 11370504 14647300 17334274 22020097 24412192 21495819 21495826 21495827 21495828 22446082 22446083 22642689 22642690 22642691 22806529 23134209 23134210 23134211 23134212 23134213 23134214 23134215 23134216 23134217 23134218 23134219 23134220 23134221 23134222 23134223 23134224 23134225 23822337 23822338 23822339 23822340 23822341 23953412 23953415 23953416 23953418 23953420 23953422 23953423 23953425 23953428 23953429 23953432 23953433 23953436 23953437 23953440 23953442 23953446 23953447 23953448 23953449 23953450 23953451 23953452 23953454 23953455 23953456 23953458 23953459 23953460 23953462 23953463 23953464 23953465 23953466 23953467 23953468 23953469 23953470 23953492 23953504 23953524 23953573 23953574 23953575 23953576 23953584 23953610 23953636 23953641 23953643 24379406 24379424 24379499 24379503 24379614 24379696 24379703 24379733 24379751 24379787 25034792 25034802 25034905 25034908 25034909 25034912 25034947 25034983 25035000 25035010 25035112 25035172 25035173 25035224 25035301 25035332 25035463 25035511 25035572 25035593 25035625 25035694 25035702 25362438 25362487 25362492 25362596 25362647 25362686 25362696 25362720 25362765 25362798 25362809 25362813 25362815 25362816 25362820 25362913 25362915 25362938 25362957 25363009 14 2009-07-31 11:41:34.023 2014-03-05 15:54:32.787 10355668 10355670 10355672 10355674 10355676 10355678 10355680 10355682 10355684 10355687 10355689 10355691 25395369 1179738 10 3637846 3637922 10 2 1212505 6848524 6848525 6848526 6848527 6848528 6848529 6848530 6848531 6848532 6848533 1343517 4849688 4849689 4849758 60 2007-06-22 13:07:08.573 2008-10-21 16:16:10.363 1179739 1179740 1179741 1179742 1179754 1179755 1179756 1179757 1179759 1179760 1179763 1179766 1179767 1179769 1179770 1179783 1179784 1179785 1179786 1179897 1179898 1179899 1179900 1179901 1179902 1179903 1179904 1179905 1179906 1179907 1179908 1179909 1179910 1179911 1179912 1179913 1179914 1179915 1179916 1179917 1179918 1179919 1179920 1179921 2162889 2162890 2162891 2162892 2162893 4456484 4456485 4456486 4456487 4456493 4456500 4915201 4915202 6750211 6750213 17236005 80 10 80 2 17268771 18777006 18777007 18777008 18777009 18777010 18777011 33 2011-05-30 20:39:26.043 2012-02-03 09:16:05.757 17236007 17236009 17236011 17236013 17236015 17236017 17236019 17236022 17236024 17236026 17236028 17236030 17236032 17236034 17236035 17236037 17236038 17236073 17236079 17236081 17236083 17236085 17236087 17236089 17236100 17236102 17236104 17236106 17236108 17236110 17236169 18579772 17170435 17170436 17170437 17170438 17170440 17170439 17170441 17170442 13336581 91 10 80 91 2 13369349 1 2010-06-09 09:14:10.053 2010-06-09 09:14:10.053 596 10 597 635 674 10912252 10 2 594 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 1343511 8 2007-01-17 10:11:40.717 2007-01-22 14:23:25.287 631 632 633 634 636 637 673 597 596 10 596 2 595 1 2007-01-17 10:16:37.923 2007-01-17 10:16:37.923 8355863 80 10356413 10 80 2 8388630 16515520 16515521 16515522 16515523 72 2009-01-05 12:16:17.787 2011-03-18 09:12:36.213 8355864 8355885 8355992 8355993 8355995 8355997 8355999 8356001 8356003 8356004 8356006 8356007 8356008 8356010 8356012 8356013 8356015 8356017 8356053 8356055 8356060 8356062 8356064 8356065 8356068 8356070 8356072 8356076 8356078 8356080 8356082 8356084 8356086 8356088 10356411 10912264 10912277 10912278 10912347 10912348 11797149 11797151 11797153 11797155 11797157 11797255 11797256 11797258 11797261 11797262 11797264 11797266 11797268 11797269 11797271 11797273 11797275 11797277 11797279 11797280 11797282 11797284 11797286 11797287 11797289 11797291 11797292 15631403 16416858 16416860 16416861 8519681 8519682 8519683 8519684 8519685 11042895 11042896 11042897 11042898 11042899 11042901 11042902 13238276 13238277 13238278 13238279 13238280 13238281 13238282 8519691 8519692 8519693 8519694 8519695 8519696 8519697 8519698 8519699 11042891 11763748 8519700 8519701 8519702 8519703 8519704 8519705 8519706 8519707 8519708 8519709 8519710 11042893 3637922 1179738 10 1179738 2 3670672 5013699 5013700 5013701 5013702 5013703 5013704 5013705 5013706 5013707 5013708 5013709 5013710 5013711 5013712 5013713 5013714 5013715 5013716 5013717 5013718 5013719 5013720 5013721 5013722 5013723 4849759 4849761 4849781 40 2008-05-07 14:59:30.507 2008-06-09 09:36:21.773 3637927 3637930 3637934 3637936 3637938 3637940 3637942 3637944 3637946 3637949 3637988 3637992 3637995 3638003 3638005 3638007 3638008 3638010 3638012 3638014 3638016 3638018 3638020 3638022 3638029 3638037 3638039 3638041 3638043 3638045 3638047 3638114 3638116 3638118 3638119 3638121 4456497 4915259 4915267 3735602 3735607 3735622 5046287 11797058 80 10 80 2 11829811 2 2010-04-26 16:05:08.450 2010-04-26 16:09:33.953 11797060 17858877 80 10 80 2 17891620 3 2011-07-15 08:16:34.157 2011-07-15 22:59:59.387 17858879 18251779 5374184 91 10 80 91 2 5406932 5505494 3 2008-06-17 16:49:15.183 2008-06-17 16:50:54.130 5374188 5374189 91 80 5374184 10911819 10911836 10911848 13336581 10 80 2 89 13434985 13434986 13434987 13434988 13434989 28 2006-10-24 21:49:36.650 2010-06-09 08:49:22.707 92 93 94 5374178 6750218 7766094 7766095 7766099 7766113 7766114 7766116 8355857 8355859 8355860 9798394 10911806 10911807 10911808 10911817 10911824 10911828 10911831 10911834 10911837 10911839 10911846 13336579 1 7897131 7897132 7897133 7897134 7897135 7897136 7897137 7897138 7897139 7897140 7897141 7897142 7897143 7897144 7897145 7897146 11042820 11042821 11042822 11042823 11042824 11042825 11042826 11042827 11042828 11042829 11042830 11042831 11042832 11042833 11042834 11042835 11042836 11042837 11042838 11042839 11042840 11042841 11042842 11042843 11042844 11042845 11042846 11042847 11042848 11042849 11042850 11042851 11042852 9797681 81 9797689 10 80 81 2 9830450 9896416 9896417 9896418 9896419 6 2009-04-14 22:56:40.067 2009-04-17 08:27:43.893 9797682 9797684 9797686 9797687 9797754 9928708 9928709 8912969 8355981 10 80 8355981 2 8945734 4 2009-01-22 10:35:49.373 2009-01-22 10:53:57.393 8912973 8912981 8912985 9797689 9797681 10 80 81 9797681 2 9830458 9896140 9896141 9896142 9896143 9896144 9896145 9896146 9896147 9896148 9896149 9896150 9896151 9896152 9896153 9896154 9896155 2 2009-04-14 23:11:31.037 2009-04-14 23:21:30.760 9797691 635 596 10 596 2 633 2831 2832 2 2007-01-19 09:03:26.357 2007-01-19 09:14:33.853 638 1179794 80 10 80 2 1212561 1246186 1246187 1246188 1343512 1343518 1343519 1343520 1343523 1343524 1343525 1343526 4849704 11 2007-07-02 09:05:44.180 2007-07-13 08:42:47.643 1179795 1179801 1179802 1179803 1179804 1179808 1179809 1179810 1179811 1179832 1540110 1540114 1540115 80 10 81 91 841 1179794 1179819 1179945 4456506 8061037 8355863 8355981 8356090 8356145 10355890 10356008 10912280 10912345 11797058 11797071 13664436 13926583 17236005 17858877 10 2 78 18777012 18777013 18777014 18777015 18777016 18777017 18777018 18777019 18777020 18777021 18777022 18777023 18777024 18777025 18777026 18777027 18777028 18777029 18777030 18777031 18777032 18777033 18777034 18777035 18777036 18777037 18777038 18777039 18777040 18777041 18777042 18777043 18777044 18777045 18777046 18777047 18777048 18777049 18777050 18777051 18777052 18777053 18777054 18777055 18777056 18777057 18777058 18777059 18777060 18777061 18777062 18777063 18777064 18777065 18777066 18777067 18777068 18777069 18777070 18777071 18777072 18777073 18777074 18777075 18777076 18777077 18777078 18777079 18777080 18777081 18777082 18777083 18777084 18777085 18777086 18777087 18777088 18777089 206 1343516 1343521 1343556 1933370 3604482 4849690 4849757 78 2006-10-24 15:12:15.247 2012-02-03 09:16:05.880 89 90 840 1179793 1179818 1179922 1179944 1179951 1179953 1179955 1179958 1179959 1179960 1179961 1179974 1179975 2162835 2162836 3114484 4456467 4456468 4456471 4456473 4456474 4456489 4456491 4456508 4915255 8061034 8061036 8061038 8355861 8355892 8355896 8355953 8356018 8356020 8356087 8356136 8356152 10355911 10355912 10912284 10912285 10912289 10912295 11797054 11797069 11797078 13664434 13664509 13926581 15204476 15630753 15630755 15630757 15630758 15630761 15630763 15630775 15630779 15630781 15630796 15630798 15630801 15630803 15630804 15630805 15630807 15630809 15630811 15630812 15631300 15631302 17236020 17858875 18579773 1540129 13860906 15728688 15728689 15728690 15728691 15728692 15728694 15728695 15728696 15728697 15728698 81 80 82 9797681 10 80 2 79 11010504 11010505 11010506 11010507 1343515 5 2006-10-24 15:13:04.743 2009-10-28 17:10:29.840 9797679 9798260 9798261 10911833 9928713 9928756 9928757 82 81 10 80 81 2 80 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 7 1343514 4 2006-10-24 15:18:56.927 2006-10-24 15:40:10.387 83 84 85 3637846 1179738 10 1179738 2 3670596 2 2008-05-06 14:31:26.493 2008-05-30 10:47:30.393 4456495 10911836 91 10 80 91 2 10944600 11010561 11010562 11010563 11010564 11010565 11010566 11010567 11010568 4 2009-10-28 17:11:13.457 2009-10-28 17:18:53.937 10911840 10911841 10911842 10356008 80 10 80 2 10388734 10461536 10461537 3 2009-08-17 09:07:20.000 2009-09-30 13:48:58.817 10356010 10356352 10911848 91 10 80 91 2 10944612 8 2009-10-29 09:35:35.460 2009-11-02 09:20:41.893 10911849 10911850 10911851 10911853 10911863 10911869 10911871 11042853 11042855 11042857 11042859 11042861 11042863 11042864 11042862 11042860 11042858 11042856 11042854 8061037 80 10 80 2 8093800 10461643 10461644 10461645 26 2008-12-18 11:24:49.237 2009-09-30 16:19:18.840 8061040 8061042 8061044 8061046 8061048 8061049 8061050 8061052 8061053 8061055 8061057 8061059 8061061 8061063 8061065 8061067 8061069 8356160 8356161 8356163 10356358 10356361 10356365 10356367 10356368 8519712 8912971 8355981 10 80 8355981 2 8945735 7 2009-01-22 10:36:03.900 2010-01-07 17:28:29.780 8912975 10911829 10911830 10911843 10912354 10912357 10911819 91 10 80 91 2 10944585 3 2009-10-28 16:52:33.203 2009-10-28 16:54:46.730 10911821 10911822 674 596 10 596 2 671 1343513 8 2007-01-22 14:36:00.070 2007-01-22 16:48:51.420 675 676 677 678 679 680 681 13664436 80 10 80 2 13697187 14024905 22 2010-06-29 09:39:57.047 2010-07-28 08:02:19.393 13664438 13664440 13664442 13664444 13664446 13664448 13664450 13664452 13664453 13664455 13664457 13664458 13664460 13664462 13664489 13664491 13664501 13926410 13926447 13926449 13926588 10 80 596 1179738 10355664 2 8 15706100 15706101 15706102 15706103 15706104 15706105 15706106 15706107 15706108 11 205 238 1343502 1343522 1343555 1933369 1933373 3276803 3276804 3276805 4390913 4849675 4849760 47 2006-10-17 21:39:33.643 2011-01-05 15:20:04.763 14 15 16 17 25 27 28 31 32 34 35 36 39 40 41 43 44 57 58 60 61 62 63 64 75 76 77 78 79 86 87 88 630 1050 1179735 1179736 1179737 3113031 4456480 4456482 4456483 4456490 4915257 5832937 6750214 15630800 8355981 80 8912969 8912971 10 80 2 8388742 11010569 11010570 11010571 11010572 19 2009-01-08 16:16:54.793 2009-10-28 17:27:41.500 8355983 8355985 8355986 8355988 8355990 8356114 8356116 8356216 8356217 8912922 8912925 8912972 8912974 8912977 8912978 8912979 8912983 10911844 1179819 80 10 80 2 1212586 1246185 13 2007-07-12 20:54:27.637 2007-07-13 00:12:01.220 1179820 1179821 1179822 1179823 1179824 1179825 1179826 1179827 1179828 1179829 1179830 1179831 1540118 25035593 0 10355664 2014-02-23 11:47:54.850 2014-02-23 11:47:54.850 8093800 {code} # It's recommended that the final XML is to be checked into the puckxml module in MBARI's CVS on moonjelly. # Submit the Metadata to SSDS using the SSDSLoads application (available at [http://new-ssds.mbari.org/ssds-docs/client/]. The \-h option provides a usage note): {code} java -jar ssdsLoads-new-ssds.jar -d 1696.xml -x # Use '-x' to test and then '-s' to submit {code} ## If a mistake is made in the metadata (e.g. forgetting the platform Deployment) it may be easier to undo the submission and start over - do a deep delete on the parent deployment if this is the case. ## Minor attribute fixes may be done by editing the database once the deployments have been loaded # Check that the metadata has been successfully loaded using the Explorer application. h2. Establish data publishing application # This step assumes that there will be some application reading data from the deployed instrument. Perhaps it is a legacy application which reads the data to load into a custom data storage or visualization system. For this application to publish data to SSDS it must have visibility of each record the instrument produces as that record needs to be "packaged" into a SensorDataPacket and "published" to SSDS. # An example Java application that will package and publish a record is below: {code} import java.io.IOException; import moos.ssds.jms.PublisherComponent; import moos.ssds.transmogrify.SSDSDevicePacket; /** *

* Publish instrument data records to the SSDS database. The client must provide * SSDS device ID, timeStamp, sequence number, and payload. *

*
* * @author : $Author: mccann $ * @version : $Revision: 1.17.2.7 $ *
*

* The * Monterey Bay Aquarium Research Institute (MBARI) provides this * documentation and code "as is", with no warranty, express * or implied, of its quality or consistency. It is provided without * support and without obligation on the part of MBARI to assist in its * use, correction, modification, or enhancement. This information * should not be published or distributed to third parties without * specific written permission from MBARI. *

*
* Copyright 2008 MBARI.
* MBARI Proprietary Information. All rights reserved.
*
*
*/ /** * @author mccann * */ public class SsdsPublisher { /** * Publish data record from an instrument to SSDS * * @param deviceID * is the SSDS Device ID for the instrument that produces the data in payload * @param epochMilliseconds * time of payload sample in milliseconds since 1/1/1970 0000 GMT * @param sequenceNumber * an incrementing number for each packet * @param payload * data from instrument */ public static void publish(long deviceID, long epochMilliseconds, long sequenceNumber, String payload) { // Create a publisher component PublisherComponent pc = new PublisherComponent(); // Create a new SensorDataPacket SSDSDevicePacket packetToSend = new SSDSDevicePacket(deviceID, payload .getBytes().length); // Set the packet type to data (0 = Metadata, 1 = Data, 2 = Message) packetToSend.setPacketType(1); // Assign the time packetToSend.setSystemTime(epochMilliseconds); // Set the parentID to 0 for a parentless deployment packetToSend.setPlatformID(0); // Set the metadataref number to 0, if the data format changes and we can // refer to a different metadata packet then this number will change. packetToSend.setMetadataRef(0); // Set the sequence number packetToSend.setSequenceNo(sequenceNumber); // Set the payload packetToSend.setDataBuffer(payload.getBytes()); // Set the record type to 1 as this is the most common situation packetToSend.setRecordType(1); try { pc.publishBytes(SSDSDevicePacket .convertToPublishableByteArray(packetToSend)); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } /** * Test of SsdsPublisher.publish() * * @param args * No arguments are taken. Test values hard coded. */ public static void main(String[] args) { /* * Example packet for EITS instrument */ long eitsDeviceID = 1696; // 1696 is actual SSDS deviceID for EITS long sampleTime = 1228431283209L; // Sample time: 2008-12-04 22:54:43 long sampleSequenceNumber = 1; // May set it to ID from DB insert // Payload values must match the RecordVariables in SSDS String samplePayload = "12.2, 4.3, 768.2, 2.3, 1.2, 0.2"; // Temp, Cond, Pres, U, V, W SsdsPublisher.publish(eitsDeviceID, sampleTime, sampleSequenceNumber, samplePayload); } } {code} # An example of a shell script (the getM1 OASIS download) calling a perl script [^ssdsSubmit.pl] to publish download statistics records to SSDS:   {code} # Record start of download set starttime_es = `perl -e 'print time, "\n"'` # Download the data and record error status $DOWNLOAD $DOWNLOAD_OPTS >& $DATAFILE set rtnsts = $status # Record the end of download set endtime_es = `perl -e 'print time, "\n"'` # Get the raw data filesize set filesize = `ls -l $DATAFILE | sed -f $BIN/fs.sed` # Log download statistics and submit to SSDS - IDs are unique for mooring deployment set deviceId = 1698 set parentId = 1306 /oasis/bin/ssdsSubmit.pl $deviceId $parentId "$starttime_es,$endtime_es,$filesize,$rtnsts" {code} # If using Java incorporate the above class into a Java application that reads the data from the instrument and calls a method like the main() example for each record # Set up the application to execute the loads for the duration of the deployment # Configure DStoNetCDF.pl cron job execution as is done for the OASIS data processing within the ssdsadmin account on elvis. (For MARS deployments this is done for the DataProducerGroup MARS2008; see the processCurrentMooringDataStreams.sh script in /u/ssdsadmin/dev/DPforSSDS/cimt/). \\ h2. Generate a packet from a file and publish to SSDS {warning:title=Under Construction} This section is still under construction, pay no attention {warning} # Download a jar file # Create the properties file like the one here: {noformat} # This file contains the information that will be used to construct a data packet to publish to SSDS. # This is a SIAM property that (in theory) would allow multiple message formats # that can be handled. Right now, there is only one and it should always be 0. DevicePacketVersion=0 # This is the ID of the source of the data (i.e. the device that generated the packet # being sent. SourceID=1 # This is the timestamp on the packet which normally represents when the packet # was created. In the system it is normally epoch seconds, but for clarity sake, # you can enter it in ISO 8601 format (http://en.wikipedia.org/wiki/ISO_8601) Timestamp=2009-09-30T15:47:00Z # This is the sequence number that will be on the packet. It should reflect the # index in the order of generation of packets from the source device. SequenceNumber=1 # This is the reference number that points to the sequence number of the packet # which contains the metadata that describes the contents of this packet. MetadataRef=0 # This is the ID of the parent device (if one exists) that the generating device # (see SourceID above) was connected to when it generated the packet ParentID=0 # This is the type of record that is being produced. Source devices can produce # multiple types of records of the same type. For example, a device can produce # data records in multiple formats. This allows you to identify which format # is used for the generated packet. It serves to link the pre-defined metadata # to the packet and allows machines and humans to understand the contents of the # packet. RecordType=0 # This value determines what type of packet (SIAM equivalent) will be constructed # using the information in this file. The available options are: # 1. METADATA # 2. SENSOR_DATA # 3. DEVICE_MESSAGE SecondStreamID=METADATA # This indicates which version of the above type of packet will be constructed. # At the time of this authoring, the only option available is 0 for all three # types listed above. In theory this would allow you to have multiple forms of # any of the above types of packets, but it has not been used to date. SecondPacketVersion=0 # This is the name of the file (located with this properties file) that contains # the payload of the first buffer. It will be read as straight bytes into a # byte buffer. NOTE: with METADATA packets the first buffer contains something # that is called 'cause' which is not used much by SSDS. FirstBuffer=test-first-buffer.txt # This is the name of the file (located with this properties file) that contains # the payload of the second buffer. It will be read as straight bytes into a # byte buffer. NOTE: with METADATA packets the second buffer is usually where the # main metadata is stored (for example, device XML). SecondBuffer=test-second-buffer.xml {noformat} # Create the two buffer files # Run using command: {noformat} the command {noformat}]]> 8061037 4849761 3 3637922 2008-06-05 13:14:15.023 2008-06-09 09:38:15.023 4849760 1 10 2008-06-05 12:57:15.033 2008-06-05 12:57:15.033 4849759 3 3637922 2008-06-05 12:57:15.030 2008-06-09 09:39:15.030 1050 1047 34 2006-10-17 21:39:33.643 2007-01-19 08:43:12.000 10 4849758 3 1179738 2008-06-05 12:57:15.027 2008-06-09 09:39:15.027 4849757 1 80 2008-06-05 12:56:15.067 2008-06-05 12:56:15.067 1212505 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms *(8 days - 8 KG)* ## (.5 day) Try to change OASIS to make mooring turns less painful (documentation basically) ## (5 days) Build non-JMS mechanism for users to send data/metadata to SSDS. ## (1 day) Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## (1 day) Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Improve Testing *(5 days - 3 KG, 2MM)* ## (.5 day) Verify (unit tests) that the RecordDescription level parse regular expression works ## (.5 day) Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## (.5 day) Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## (.5 day) Write valid unit test for Object and XMLBuilders ## (.5 day) Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## (.5 day) Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. ## (2 days) Improve Wrapper tests # Documentation *(6 days - 3 KG, 3 MM)* ## (.5 day) Put UML diagram of data model on developer section of web app. ## (.5 day) Finish documenting data packet structure on web pages. ## (5 days) Document Explorer, Admin app and HOOVES h3. Tasks that were descoped from 900823 SSDS Hardening Proposal # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). # Load historical OASIS data into SSDS (data and metadata). *This is important but too big for this, separate project* ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). *IMPORTANT EVEN THOUGH DESCOPED* ]]> 1179738 4849781 1 3637922 2008-06-09 09:37:15.037 2008-06-09 09:37:15.037 24412192 1 10355664 2014-01-31 06:20:15.027 2014-01-31 06:20:15.027 9 2 7 3 2006-10-17 21:39:33.420 2008-05-30 10:50:44.747 18 4456499 8290315 13926627 13959389 2 2010-07-27 14:50:24.337 2010-07-27 16:07:25.760 13926583 25035625 0 10355664 2014-02-23 19:58:01.153 2014-02-23 19:58:01.153 13926586 13959348 1 2010-07-27 14:50:24.337 2010-07-27 14:50:24.337 13926583 13926581 13959343 52 2006-10-24 15:12:15.247 2010-07-01 06:20:26.887 80 11240426 11273163 7 2010-03-10 21:41:49.343 2010-03-10 22:16:49.690 11240414 11240424 11273161 6 2010-03-10 21:41:49.343 2010-03-10 22:10:12.997 11240414 13926588 13959350 21 2010-06-29 09:39:57.047 2010-07-09 22:15:39.443 13664436 11240421 11273158 4 2010-03-10 21:41:49.343 2010-03-10 21:57:51.807 11240414 11240422 11273159 5 2010-03-10 21:41:49.343 2010-03-10 22:09:00.093 11240414 11240419 11273156 3 2010-03-10 21:41:49.343 2010-03-10 21:53:02.957 11240414 11240417 11273154 2 2010-03-10 21:41:49.343 2010-03-10 21:45:34.463 11240414 11240415 11273152 1 2010-03-10 21:41:49.343 2010-03-10 21:41:49.343 11240414 11240412 11273149 39 2007-02-08 12:50:50.907 2009-07-16 18:00:08.287 841 18251779 18284547 2 2011-07-15 08:16:34.157 2011-07-15 08:27:43.657 17858877 4849704 1 1179794 2008-06-03 13:53:15.023 2008-06-03 13:53:15.023 4849690 5 80 2008-06-03 13:17:15.037 2008-06-05 12:56:15.063 25035694 0 10355664 2014-02-24 14:01:09.170 2014-02-24 14:01:09.170 9896147 9797689 2009-04-14 23:21:30.790 2009-04-14 23:21:30.790 9896146 9797689 2009-04-14 23:21:30.790 2009-04-14 23:21:30.790 9896145 9797689 2009-04-14 23:21:30.790 2009-04-14 23:21:30.790 9896144 9797689 2009-04-14 23:21:30.790 2009-04-14 23:21:30.790 9896143 9797689 2009-04-14 23:21:30.790 2009-04-14 23:21:30.790 9896142 9797689 2009-04-14 23:21:30.790 2009-04-14 23:21:30.790 9896141 9797689 2009-04-14 23:21:30.790 2009-04-14 23:21:30.790 9896140 9797689 2009-04-14 23:21:30.790 2009-04-14 23:21:30.790 25035702 0 10355664 2014-02-24 17:51:04.690 2014-02-24 17:51:04.690 180 82 2006-10-24 15:40:10.410 2006-10-24 15:40:10.410 9896152 9797689 2009-04-14 23:21:30.790 2009-04-14 23:21:30.790 13926447 13959214 19 2010-06-29 09:39:57.047 2010-07-07 12:30:20.533 13664436 181 82 2006-10-24 15:40:10.410 2006-10-24 15:40:10.410 9896153 9797689 2009-04-14 23:21:30.790 2009-04-14 23:21:30.790 182 82 2006-10-24 15:40:10.410 2006-10-24 15:40:10.410 9896154 9797689 2009-04-14 23:21:30.790 2009-04-14 23:21:30.790 13926449 13959216 20 2010-06-29 09:39:57.047 2010-07-09 21:46:59.717 13664436 9896155 9797689 2009-04-14 23:21:30.790 2009-04-14 23:21:30.790 9896148 9797689 2009-04-14 23:21:30.790 2009-04-14 23:21:30.790 9896149 9797689 2009-04-14 23:21:30.790 2009-04-14 23:21:30.790 9896150 9797689 2009-04-14 23:21:30.790 2009-04-14 23:21:30.790 9896151 9797689 2009-04-14 23:21:30.790 2009-04-14 23:21:30.790 13959345 13926583 24379406 0 10355664 2014-01-28 05:49:57.870 2014-01-28 05:49:57.870 7897131 91 2008-11-14 10:00:11.220 2008-11-18 16:42:44.010 15797 13 1212561 1179794 7897133 91 2008-11-14 10:00:11.220 2008-11-14 10:04:57.857 5061 2 7897131 7897132 91 2008-11-14 10:00:11.220 2008-11-14 10:00:11.220 133 1 7897131 11 2 10 2006-10-25 21:59:15.057 2006-11-09 15:00:15.027 179 82 2006-10-24 15:40:10.410 2006-10-24 15:40:10.410 7897142 91 2008-11-14 10:00:11.220 2008-11-18 16:34:00.283 14876 11 7897131 7 3 82 2006-10-24 15:29:15.043 2007-06-22 13:12:15.057 178 82 2006-10-24 15:40:10.410 2006-10-24 15:40:10.410 7897143 91 2008-11-14 10:00:11.220 2008-11-18 16:37:23.663 16136 12 7897131 177 82 2006-10-24 15:40:10.410 2006-10-24 15:40:10.410 7897144 91 2008-11-18 16:51:09.747 2008-11-18 17:13:47.137 15473 3 176 82 2006-10-24 15:40:10.410 2006-10-24 15:40:10.410 7897145 91 2008-11-18 16:51:09.747 2008-11-18 16:51:09.747 133 1 7897144 175 82 2006-10-24 15:40:10.410 2006-10-24 15:40:10.410 7897146 91 2008-11-18 16:51:09.747 2008-11-18 16:58:22.057 7301 2 7897144 174 82 2006-10-24 15:40:10.410 2006-10-24 15:40:10.410 173 82 2006-10-24 15:40:10.410 2006-10-24 15:40:10.410 172 82 2006-10-24 15:40:10.410 2006-10-24 15:40:10.410 171 82 2006-10-24 15:40:10.410 2006-10-24 15:40:10.410 7897134 91 2008-11-14 10:00:11.220 2008-11-14 10:05:28.203 5132 3 7897131 170 82 2006-10-24 15:40:10.410 2006-10-24 15:40:10.410 7897135 91 2008-11-14 10:00:11.220 2008-11-14 10:29:50.913 10351 4 7897131 169 82 2006-10-24 15:40:10.410 2006-10-24 15:40:10.410 7897136 91 2008-11-14 10:00:11.220 2008-11-14 10:30:12.837 10663 5 7897131 168 82 2006-10-24 15:40:10.410 2006-10-24 15:40:10.410 7897137 91 2008-11-14 10:00:11.220 2008-11-14 10:32:21.777 11395 6 7897131 167 82 2006-10-24 15:40:10.410 2006-10-24 15:40:10.410 7897138 91 2008-11-14 10:00:11.220 2008-11-14 11:03:21.593 12233 7 7897131 7897139 91 2008-11-14 10:00:11.220 2008-11-14 11:04:55.517 12231 8 7897131 7897140 91 2008-11-14 10:00:11.220 2008-11-14 12:53:34.467 12454 9 7897131 7897141 91 2008-11-14 10:00:11.220 2008-11-18 13:16:46.150 14607 10 7897131 1246188 1179794 2007-07-13 08:42:47.693 2007-07-13 08:42:47.693 1246187 1179794 2007-07-13 08:42:47.693 2007-07-13 08:42:47.693 1212586 1179819 1246186 1179794 2007-07-13 08:42:47.693 2007-07-13 08:42:47.693 1246185 1179819 2007-07-13 00:12:01.257 2007-07-13 00:12:01.257 24379424 0 10355664 2014-01-29 18:10:08.843 2014-01-29 18:10:08.843 13926410 13959178 18 2010-06-29 09:39:57.047 2010-06-30 17:08:32.583 13664436 13697187 13664436 5832937 5865695 44 2006-10-17 21:39:33.643 2008-06-05 12:56:39.363 10 5832936 5865694 9 2007-02-08 12:50:50.907 2008-05-30 10:08:07.320 841 10356436 10389148 4 2009-10-07 10:47:14.877 2009-10-07 11:25:27.023 10356413 10356453 10389165 6 2009-10-07 10:47:14.877 2009-10-07 16:33:35.207 10356413 10356455 10389167 7 2009-10-07 10:47:14.877 2009-10-07 22:44:25.357 10356413 10356446 10389158 5 2009-10-07 10:47:14.877 2009-10-07 13:31:04.797 10356413 5832941 5865699 11 2007-02-08 12:50:50.907 2008-08-13 09:08:36.707 841 5832939 5865697 10 2007-02-08 12:50:50.907 2008-08-13 09:03:35.090 841 4456500 4489266 55 2007-06-22 13:07:08.573 2008-05-30 10:44:28.353 1179738 4456491 4489258 26 2006-10-24 15:12:15.247 2008-05-30 10:38:05.493 80 4456493 4489260 54 2007-06-22 13:07:08.573 2008-05-30 10:25:53.807 1179738 4456495 4489261 1 2008-05-06 14:31:26.493 2008-05-30 10:47:17.760 3637846 4456497 4489263 37 2008-05-07 14:59:30.507 2008-05-30 10:48:13.477 3637922 24379499 0 10355664 2014-02-03 07:38:04.443 2014-02-03 07:38:04.443 10356420 10389132 3 2009-10-07 10:47:14.877 2009-10-07 11:16:06.743 10356413 4456508 4489272 27 2006-10-24 15:12:15.247 2008-05-30 10:41:18.093 80 4456509 4489273 1 2008-05-30 15:28:37.337 2008-05-30 15:28:37.337 4456506 10356411 10389123 35 2009-01-05 12:16:17.787 2009-01-12 13:08:40.227 8355863 24379503 0 10355664 2014-02-03 11:27:20.730 2014-02-03 11:27:20.730 10356417 10389129 2 2009-10-07 10:47:14.877 2009-10-07 10:51:28.817 10356413 10356415 10389127 1 2009-10-07 10:47:14.877 2009-10-07 10:47:14.877 10356413 4456473 4489241 23 2006-10-24 15:12:15.247 2008-05-30 09:48:24.970 80 4456474 4489242 24 2006-10-24 15:12:15.247 2008-05-30 09:53:09.160 80 4456471 4489239 22 2006-10-24 15:12:15.247 2008-05-30 09:47:48.627 80 4456469 4489237 6 2007-02-08 12:50:50.907 2007-02-09 09:47:56.587 841 4456467 4489235 20 2006-10-24 15:12:15.247 2008-04-02 11:53:50.703 80 4456468 4489236 21 2006-10-24 15:12:15.247 2008-05-30 09:46:51.990 80 10356367 10389080 24 2008-12-18 11:24:49.237 2009-09-30 15:18:07.117 8061037 10356368 10389081 25 2008-12-18 11:24:49.237 2009-09-30 16:13:45.840 8061037 10911869 10944633 6 2009-10-29 09:35:35.460 2009-10-29 10:23:29.053 10911848 10911871 10944635 7 2009-10-29 09:35:35.460 2009-10-29 10:28:51.030 10911848 10356365 10389078 23 2008-12-18 11:24:49.237 2009-09-30 14:52:20.990 8061037 4456490 4489257 42 2006-10-17 21:39:33.643 2008-05-30 10:13:26.143 10 1343556 4 80 2007-08-06 10:32:15.023 2007-08-06 10:38:15.027 4456489 4489256 25 2006-10-24 15:12:15.247 2008-05-30 09:54:34.617 80 23822337 0 10355664 2013-11-07 06:16:55.660 2013-11-07 06:16:55.660 1343555 1 10 2007-08-06 10:31:15.097 2007-08-06 10:31:15.097 4456487 4489254 53 2007-06-22 13:07:08.573 2008-05-30 10:25:22.583 1179738 23822339 0 10355664 2013-11-08 16:09:08.557 2013-11-08 16:09:08.557 4456486 4489253 52 2007-06-22 13:07:08.573 2008-05-30 10:16:22.557 1179738 10911863 10944627 5 2009-10-29 09:35:35.460 2009-10-29 10:06:04.880 10911848 23822338 0 10355664 2013-11-08 14:43:19.753 2013-11-08 14:43:19.753 4456485 4489252 51 2007-06-22 13:07:08.573 2008-05-30 10:14:27.370 1179738 23822341 0 10355664 2013-11-13 21:16:23.483 2013-11-13 21:16:23.483 4456484 4489251 50 2007-06-22 13:07:08.573 2007-11-14 06:46:29.933 1179738 4456483 4489250 41 2006-10-17 21:39:33.643 2008-05-30 10:12:37.997 10 23822340 0 10355664 2013-11-11 11:28:08.327 2013-11-11 11:28:08.327 4456482 4489249 40 2006-10-17 21:39:33.643 2008-05-30 10:11:09.467 10 10911851 10944615 3 2009-10-29 09:35:35.460 2009-10-29 09:42:10.737 10911848 4456480 4489247 39 2006-10-17 21:39:33.643 2008-01-10 05:40:42.947 10 10911853 10944617 4 2009-10-29 09:35:35.460 2009-10-29 09:44:49.860 10911848 1343552 1 1179945 2007-08-03 14:00:15.040 2007-08-03 14:00:15.040 4456478 4489245 8 2007-02-08 12:50:50.907 2008-05-30 10:07:33.783 841 4456477 4489244 7 2007-02-08 12:50:50.907 2008-05-30 09:47:48.633 841 13336579 13369347 27 2006-10-24 21:49:36.650 2009-10-29 09:30:52.963 91 1343543 4 841 2007-07-13 15:28:15.047 2007-10-04 11:50:15.037 11010561 10911836 2009-10-28 17:18:53.953 2009-10-28 17:18:53.953 11010563 10911836 2009-10-28 17:18:53.953 2009-10-28 17:18:53.953 11010562 10911836 2009-10-28 17:18:53.953 2009-10-28 17:18:53.953 1343522 1 10 2007-07-03 10:24:15.027 2007-07-03 10:24:15.027 11010568 10911836 2009-10-28 17:18:53.953 2009-10-28 17:18:53.953 11010569 8355981 2009-10-28 17:27:41.543 2009-10-28 17:27:41.543 10356361 10389074 22 2008-12-18 11:24:49.237 2009-09-30 14:18:30.187 8061037 1343523 1 1179794 2007-07-03 10:32:15.017 2007-07-03 10:32:15.017 11010570 8355981 2009-10-28 17:27:41.543 2009-10-28 17:27:41.543 1343524 1 1179794 2007-07-03 10:33:15.027 2007-07-03 10:33:15.027 1212710 tag. For instance any nominalLat/Lon/Depth. {color:#ff0000}The one exception is the nominalDepth, if it is known please set it{color}. # If the tag has a 'name' attribute, make sure it does not have any deployment specific information in it. For example, 'ISUS Deployment' is better than 'M2 ISUS Deployment'. The reason for removing any deployment information from the XML is so that when the device moves to a different mooring, the user should not have to edit the XML. The goal is to get all the XML to a point where it never needs to be edited when an instrument is deployed (unless something in the way the data stream is generated from the instrument changes). # Go to the SSDS Device pages and verify that the all the device information (mfg, model, serial number, name, type, etc.) matches what is currently in SSDS. If any of those are different it will update the device information in SSDS when the XML comes in the data stream. # Verify RecordDescription and RecordVariables look correct. I usually go to the raw data pages in SSDS and bring up the last few packets from the device just to verify that the number of columns and bufferSeparator look about right. # Check any changes to the XML back into CVS. # Copy the XML to the \\Tornado\ssdsdata\mooring(m1\|m2)\YYYY\xml directory # Go to the \\Tornado\ssdsdata\mooring(m1\|m2)\YYYY\cfg directory. # This next step is the one that needs to be timed with the mooring turn. When the old instrument is shutdown: ## Open the ssds.cfg file in a text editor ## Find the line that shows the currently deployed instrument and copy it to a line just below it. For example, if we are replacing the GPS, it might look like this before: {panel:title=Before Copy} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} and this after: {panel:title=After Copy} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # Now change the new line to have the {color:#cc0000}{+}correct device ID{+}{color} *and* the {color:#cc0000}{+}correct XML{+}{color} file URL {panel:title=After Device ID update} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = GPS_TYPE3,1511,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1511.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # To clean up the previous deployment information, put start and end dates after the XML URL {panel:title=After Adding Start/End dates} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,2007/04/25 16:21:58,2007/08/01 10:00:00,TransformGPS instrument = GPS_TYPE3,1511,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1511.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} (Be careful on the format for the start and end date strings.  Leading 0s are required.) # Save the cfg file. {note:title=Saving the file will make the change take hold}When the ssds.cfg file changes (saved) is when the OASIS2SSDS processing will pick up the instrument change. Now, the next time it runs it will pick up the instrument change, grab the XML file from the 'xml' directory and publish it to SSDS. It will then publish all data under the new device ID. {note} # Edit the metadata in SSDS to put a close date on the old instrument deployment in SSDS. Currently I do that using Enterprise Manager. h5. If this is being done after the fact the next steps will also need to be taken. # After the new deployment shows up in SSDS (which can take up to 5 minutes after the OASIS2SSDS has completed), the start time for the new deployment will need to be edited to match the actual time the instrument was swapped. # Also because the data was being published under the incorrect device ID, it will need to be moved from one database table in SSDS_Data on Solstice to another table. ## The first thing that I do is grab the timestamp from the last packet sent from the old device in the raw data page on SSDS. ## For example, I go to: [http://new-ssds.mbari.org:8080/ssds/siamRawDataStep1.jsp] and enter the old device ID and set the number of packets back to make sure it goes far enough back to cover the actual time of the instrument swap. Then click on 'Next->'. ## Once the raw data shows up, find the last packet from the old device and grab the 'SIAM Timestamp' value (not the date/time) as that will be used in the Enterprise Manager query. ## Open Enterprise Manager and navigate to the 'SSDS_Data' database on Solstice. ## Browse the tables and find the table with the device ID of the old device and right click on it and select 'Open Table->Return all rows'. ## Click on the 'SQL' button in Enterprise Manager to bring up the SQL pane. It should show the basic query which should look something like this: {noformat} SELECT * FROM [1416] {noformat} ## Now add the where clause to pick only the data that is after the timestamp you grabbed from the last packet on the web page. {note:title=Timestamps in SQL are in Seconds}A quick note here, the 'SIAM Timestamp' on the raw data page is actually in milliseconds and the database column is in seconds so you will have to remove the last three digits of the 'SIAM Timestamp' before putting it in this query. {note} {noformat} SELECT * FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ## Run this query by clicking the run button '\!' in Enterprise Manager. ## Look over the results to make sure they look about right (usually you are looking for the length of the return which should be much shorter). You can actually use a count query to see how many rows this query will return. A count query would look like: {noformat} SELECT count(*) FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ## Once you know the query is correct (also compare sequence number in query return and the raw data page), copy it to the clipboard and close the query window in Enterprise Manager. ## Navigate to the table of the device you want to copy the data into and right click and select 'All Tasks->Import Data...' which will fire up the DTS wizard. ### Click on 'Next>' ### For the Data Source database choose Solstice ### Select the 'SSDS_Data' database (note you should have permissions to do all this and use your windows authentication) ### Click on 'Next>' ### The destination configuration should be already to go (Solistice and SSDS_Data database). ### Click on 'Next>' ### Select 'Use a query to specify the data to transfer' ### Click on 'Next>' ### Paste the query from your clipboard into the 'Query Statement' window (You can click on 'Parse' if you want a quick sanity check) ### Click on 'Next>' ### Click on the 'Results' entry under the 'Destination' column which will enable a drop down box. ### Choose the table of the newly installed device where you will be copying the data to. ### Click on 'Next>' ### Click on 'Next>' ### Click on 'Finish' which will copy the data. ### Once that is done, open the table of the old instrument and the SQL pane so that we can construct the delete query on the old data. ### Paste in the select query and verify it is the same data you copied over: {noformat} SELECT * FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ### If it looks good, click on the 'Change Query type ...' button in Enterprise Manager and select 'Delete'. This will change the query to a delete query. ### Run the query by click on the run '\!' button. That will remove all the data from the old instrument. That's it ... whew\! Kevin Gomes (August 3, 2007)]]> 1179945 11010571 8355981 2009-10-28 17:27:41.543 2009-10-28 17:27:41.543 1343525 1 1179794 2007-07-03 11:15:15.020 2007-07-03 11:15:15.020 10356358 10389071 21 2008-12-18 11:24:49.237 2009-01-13 15:24:23.830 8061037 11010564 10911836 2009-10-28 17:18:53.953 2009-10-28 17:18:53.953 1343526 5 1179794 2007-07-03 11:27:15.023 2007-07-03 14:13:15.047 13238277 8355863 2009-01-05 12:16:39.663 2010-06-08 10:16:37.033 16076 13 8519681 11010565 10911836 2009-10-28 17:18:53.953 2009-10-28 17:18:53.953 13238276 8355863 2009-01-05 12:16:39.663 2010-01-05 12:20:25.837 17424 12 8519681 11010566 10911836 2009-10-28 17:18:53.953 2009-10-28 17:18:53.953 11010567 10911836 2009-10-28 17:18:53.953 2009-10-28 17:18:53.953 1343514 3 82 2007-07-03 09:51:15.017 2007-10-04 11:51:15.027 13238281 8355863 2009-01-05 12:16:39.663 2010-06-08 10:24:05.257 15136 17 8519681 1343515 6 81 2007-07-03 09:51:15.043 2007-10-04 11:51:15.030 13238280 8355863 2009-01-05 12:16:39.663 2010-06-08 10:22:31.793 15237 16 8519681 10356352 10389065 2 2009-08-17 09:07:20.000 2009-08-17 09:13:24.580 10356008 1343516 1 80 2007-07-03 09:51:15.050 2007-07-03 09:51:15.050 13238279 8355863 2009-01-05 12:16:39.663 2010-06-08 10:20:02.247 14700 15 8519681 1343517 2 1179738 2007-07-03 09:53:15.023 2007-07-03 10:24:15.023 13238278 8355863 2009-01-05 12:16:39.663 2010-06-08 10:17:37.123 16555 14 8519681 11010572 8355981 2009-10-28 17:27:41.543 2009-10-28 17:27:41.543 1343518 3 1179794 2007-07-03 09:56:15.020 2007-07-03 14:15:15.013 1343519 2 1179794 2007-07-03 09:56:15.020 2007-07-03 10:00:15.090 1343520 1 1179794 2007-07-03 10:00:15.090 2007-07-03 10:00:15.090 1343521 1 80 2007-07-03 10:24:15.023 2007-07-03 10:24:15.023 13238282 8355863 2009-01-05 12:16:39.663 2010-06-08 10:29:48.487 21036 18 8519681 23134217 0 10355664 2013-10-23 06:33:04.030 2013-10-23 06:33:04.030 10911808 10944575 18 2006-10-24 21:49:36.650 2009-10-27 10:16:33.617 91 10911807 10944574 17 2006-10-24 21:49:36.650 2009-10-27 10:16:01.557 91 23134216 0 10355664 2013-10-18 13:05:35.467 2013-10-18 13:05:35.467 23134215 0 10355664 2013-10-18 11:12:51.047 2013-10-18 11:12:51.047 23134214 0 10355664 2013-10-15 18:26:21.390 2013-10-15 18:26:21.390 23134221 0 10355664 2013-10-25 06:25:32.853 2013-10-25 06:25:32.853 23134220 0 10355664 2013-10-24 18:41:48.100 2013-10-24 18:41:48.100 10911806 10944573 16 2006-10-24 21:49:36.650 2009-05-11 09:18:18.620 91 23134219 0 10355664 2013-10-24 17:58:08.233 2013-10-24 17:58:08.233 23134218 0 10355664 2013-10-23 17:20:20.880 2014-01-28 09:07:42.490 23134225 0 10355664 2013-10-29 02:19:38.327 2013-10-29 02:19:38.327 15631403 15664154 68 2009-01-05 12:16:17.787 2010-05-05 22:44:20.117 8355863 23134224 0 10355664 2013-10-28 21:51:25.183 2013-10-28 21:51:25.183 23134223 0 10355664 2013-10-27 05:12:24.757 2013-10-27 05:12:24.757 10911817 10944584 19 2006-10-24 21:49:36.650 2009-10-27 10:17:09.213 91 23134222 0 10355664 2013-10-26 10:36:44.287 2013-10-26 10:36:44.287 4489270 4456506 23134209 0 10355664 2013-10-06 11:08:48.603 2013-10-06 11:08:48.603 23134212 0 10355664 2013-10-09 20:23:47.617 2013-10-09 20:23:47.617 10388392 10355664 23134213 0 10355664 2013-10-14 21:06:42.687 2013-10-14 21:06:42.687 23134210 0 10355664 2013-10-06 12:49:38.537 2013-11-18 14:16:35.127 23134211 0 10355664 2013-10-08 17:59:34.370 2013-10-08 17:59:34.370 10911842 10944606 3 2009-10-28 17:11:13.457 2009-10-28 17:17:40.047 10911836 10911841 10944605 2 2009-10-28 17:11:13.457 2009-10-28 17:17:05.233 10911836 10911840 10944604 1 2009-10-28 17:11:13.457 2009-10-28 17:11:13.457 10911836 10911839 10944603 25 2006-10-24 21:49:36.650 2009-10-28 17:12:20.553 91 24379614 0 10355664 2014-02-07 03:10:27.003 2014-02-07 03:10:27.003 10911837 10944601 24 2006-10-24 21:49:36.650 2009-10-28 17:11:04.257 91 10945035 10912280 10911850 10944614 2 2009-10-29 09:35:35.460 2009-10-29 09:41:40.960 10911848 3276803 1 10 2008-01-04 14:03:15.130 2008-01-04 14:03:15.130 10911849 10944613 1 2009-10-29 09:35:35.460 2009-10-29 09:35:35.460 10911848 3276805 1 10 2008-01-04 15:03:15.047 2008-01-04 15:03:15.047 3276804 1 10 2008-01-04 14:03:15.163 2008-01-04 14:03:15.163 10911846 10944610 26 2006-10-24 21:49:36.650 2009-10-28 17:16:23.360 91 10911844 10944608 18 2009-01-08 16:16:54.793 2009-01-22 10:41:03.617 8355981 10911843 10944607 4 2009-01-22 10:36:03.900 2009-10-28 17:07:49.307 8912971 9896418 9797681 2009-04-17 08:27:43.967 2009-04-17 08:27:43.967 9896419 9797681 2009-04-17 08:27:43.967 2009-04-17 08:27:43.967 9896416 9797681 2009-04-17 08:27:43.967 2009-04-17 08:27:43.967 10911824 10944590 20 2006-10-24 21:49:36.650 2009-10-28 16:42:45.317 91 9896417 9797681 2009-04-17 08:27:43.967 2009-04-17 08:27:43.967 10911821 10944587 1 2009-10-28 16:52:33.203 2009-10-28 16:52:33.203 10911819 10911822 10944588 2 2009-10-28 16:52:33.203 2009-10-28 16:54:28.087 10911819 10911833 10944598 4 2006-10-24 15:13:04.743 2009-05-06 08:26:01.893 81 10911834 10944599 23 2006-10-24 21:49:36.650 2009-10-28 17:08:32.223 91 10911831 10944596 22 2006-10-24 21:49:36.650 2009-10-28 17:00:39.297 91 17334274 1 10355664 2011-05-27 07:33:15.090 2011-05-27 07:33:15.090 10911829 10944594 2 2009-01-22 10:36:03.900 2009-01-22 10:37:14.180 8912971 10911830 10944595 3 2009-01-22 10:36:03.900 2009-10-28 17:05:32.667 8912971 10911828 10944593 21 2006-10-24 21:49:36.650 2009-10-28 16:56:31.930 91 13926683 13959445 4 2010-07-27 14:50:24.337 2010-07-29 16:04:55.880 13926583 13926661 13959423 3 2010-07-27 14:50:24.337 2010-07-28 12:12:24.217 13926583 10945092 10912345 11763748 8355863 2009-01-12 11:13:21.350 2009-01-12 12:33:42.333 20815 11 8519700 24379696 0 10355664 2014-02-08 18:58:13.017 2014-02-08 18:58:13.017 25362492 0 10355664 2014-02-26 06:58:29.753 2014-02-26 06:58:29.753 13664491 13697242 16 2010-06-29 09:39:57.047 2010-06-30 14:32:09.443 13664436 25362487 0 10355664 2014-02-26 05:35:23.037 2014-02-26 05:35:23.037 10354987 10387742 9 2007-08-03 07:17:05.643 2009-04-01 11:24:48.407 1179945 25035112 0 10355664 2014-02-17 08:36:11.673 2014-02-17 08:36:11.673 22642689 0 10355664 2013-09-14 16:20:08.520 2013-09-14 16:20:08.520 13664501 13697252 17 2010-06-29 09:39:57.047 2010-06-30 14:34:57.570 13664436 8 10 22642690 0 10355664 2013-09-16 06:50:08.517 2013-09-16 06:50:08.517 22642691 0 10355664 2013-09-24 03:29:14.763 2013-09-24 03:29:14.763 13664509 13697260 51 2006-10-24 15:12:15.247 2010-06-29 09:30:58.793 80 8388742 8355981 17236073 17268838 18 2011-05-30 20:39:26.043 2011-05-31 12:34:13.060 17236005 3011 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 17236013 17268779 4 2011-05-30 20:39:26.043 2011-05-30 21:59:01.810 17236005 3010 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 3009 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 17236011 17268777 3 2011-05-30 20:39:26.043 2011-05-30 21:34:24.167 17236005 3008 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 3007 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 17236017 17268783 6 2011-05-30 20:39:26.043 2011-05-30 22:49:38.937 17236005 3006 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 3005 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 17236015 17268781 5 2011-05-30 20:39:26.043 2011-05-30 22:10:28.103 17236005 3004 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 3003 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 3002 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 17236022 17268788 8 2011-05-30 20:39:26.043 2011-05-30 23:11:46.283 17236005 3001 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 17236019 17268785 7 2011-05-30 20:39:26.043 2011-05-30 22:55:27.077 17236005 3000 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 17236020 17268786 75 2006-10-24 15:12:15.247 2011-02-03 09:00:15.857 80 2999 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 15631302 15664052 74 2006-10-24 15:12:15.247 2011-02-03 08:58:49.733 80 2998 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 17236026 17268792 10 2011-05-30 20:39:26.043 2011-05-31 09:39:37.650 17236005 2997 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 15631300 15664050 73 2006-10-24 15:12:15.247 2011-01-05 15:33:51.610 80 2996 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 17236024 17268790 9 2011-05-30 20:39:26.043 2011-05-31 08:35:18.723 17236005 17236030 17268796 12 2011-05-30 20:39:26.043 2011-05-31 10:35:30.917 17236005 17236028 17268794 11 2011-05-30 20:39:26.043 2011-05-31 09:40:40.743 17236005 17236034 17268800 14 2011-05-30 20:39:26.043 2011-05-31 11:53:40.090 17236005 3020 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 17236032 17268798 13 2011-05-30 20:39:26.043 2011-05-31 10:47:50.617 17236005 3018 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 17236038 17268804 17 2011-05-30 20:39:26.043 2011-05-31 12:23:05.953 17236005 3019 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 17236037 17268803 16 2011-05-30 20:39:26.043 2011-05-31 12:10:31.200 17236005 3016 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 3017 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 17236035 17268801 15 2011-05-30 20:39:26.043 2011-05-31 12:09:27.737 17236005 3014 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 3015 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 3012 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 15204476 15237240 53 2006-10-24 15:12:15.247 2010-07-27 14:33:07.063 80 3013 596 2007-01-22 14:23:25.330 2007-01-22 14:23:25.330 1179739 1212506 1 2007-06-22 13:07:08.573 2007-06-22 13:07:08.573 1179738 78 80 1179740 1212507 2 2007-06-22 13:07:08.573 2007-06-22 13:10:48.547 1179738 1179741 1212508 3 2007-06-22 13:07:08.573 2007-06-22 13:39:28.773 1179738 1179742 1212509 4 2007-06-22 13:07:08.573 2007-06-22 13:52:47.273 1179738 80 channel=0, temp=48.51, vcc=3.2316, txBiasI=3.9796, txPower=0.4652, rxPower=0.0717, status=0x0 channel=3, temp=51.10, vcc=3.2252, txBiasI=4.4665, txPower=0.5313, rxPower=0.1337, status=0x0}} ### Medusa (*SSDS ID 1485*): Looks like the previous (might have actually replaced that one because it has current data). {{-> channel=0, temp=51.69, vcc=3.2208, txBiasI=4.2839, txPower=-3.3320, rxPower=-10.6312, status=0x0 channel=3, temp=54.25, vcc=3.2142, txBiasI=4.7679, txPower=-2.7678, rxPower=-8.7681, status=0x0}} ### Digital Camera #### Thumbnail page of all images from the camera (click on image to get full image) #### Sounds like images will come back as encapsulated JPG ## CTD (SBE 16+) #### Plot raw data like on CIMT #### Process data to calculate salinity #### Plot salinity along with raw ### Current Meter, ADCP, RDI Sentinel #### John R thought the processing would be very similar to the ADCP's we have seen on MTM and M0. ### Current Meter, profiling, Aquadaopp HR Profiler, 6000m #### This is a complicated vector plot that will be calculated by matlab script. Assume we will just display the image ### Flourometer/Backscatter, Wetlabs ECO BB2F triplett #### Unknown, but assuming this would be a similar process that Reiko is running for M0 ### Oxygen, Aanderaa #### Unknown ### Turbidity, Nephalometer, C-star, 6000m #### Unknown ## MSE Axis BIN Medusa subnode 2 (*SSDS ID 1409*): Again, just message packets. ### MSP430 (*SSDS ID 1413*): Again, another MSP. Nothing new and has current data. ### MBARI Medusa (*SSDS ID 1442*): No current data, but recent data looks like {{-> channel=3, temp=49.40, vcc=3.2308, txBiasI=2.6144, txPower=0.5348, rxPower=0.2863, status=0x0}} ### Medusa (*SSDS ID 1486*): {{-> channel=3, temp=50.25, vcc=3.2476, txBiasI=2.6434, txPower=-2.6995, rxPower=-5.3036, status=0x0}} ### Digital Camera #### Thumbnail page of all images from the camera (click on image to get full image) #### Sounds like images will come back as encapsulated JPG ### CTD (SBE 16+) #### Plot raw data like on CIMT #### Process data to calculate salinity #### Plot salinity along with raw ### Current Meter, ADCP, RDI Sentinel #### John R thought the processing would be very similar to the ADCP's we have seen on MTM and M0. ### Current Meter, profiling, Aquadaopp HR Profiler, 6000m #### This is a complicated vector plot that will be calculated by matlab script. Assume we will just display the image ### Flourometer/Backscatter, Wetlabs ECO BB2F triplett #### Unknown, but assuming this would be a similar process that Reiko is running for M0 ### Oxygen, Aanderaa #### Unknown ### Turbidity, Nephalometer, C-star, 6000m #### Unknown ### Vertical Profiler, McLane(FSI velocity, Seapoint OBS, Wetlabs FL, CT&P) #### Unknown ## Sediment Trap, Honjo ### Unknown ## Sediment Trap, IRS ### Unknown # Outstanding Issues: ## Instrument questions: ### ASIMET LWR (John Ryan): Is there any post processing of the raw data and if so, what does it consist of? ### ASIMET Radiometer, short-wave (John Ryan): Is there any post processing of the raw data and if so, what does it consist of? ### ASIMET Relative Humidity and Temperature (Francisco Chavez): Is there any post processing of the raw data and if so, what does it consist of? ### ASIMET Wind Speed Direction (Francisco Chavez): Is there any post processing of the raw data and if so, what does it consist of? ### Power Can (Ed Mellinger): Any new features/graphs ### Load Cell(Lance McBride): Any processing, any new plots? ### Seahorse(Andy Hamilton): Any processing, feedback or data plots from seahorse? Any linked images? Do you want processing and products tracked? ### MBARI Medusa(Mark Chaffey): Do we want plots of any of this data? Any post processing (and resubmit/track)? ### ASIMET Barometric(Chavez or Ryan): Do plots of raw data make sense? Any post processing (and resubmit/track)? ### All Satlantic Hyperspectral Radiometers (Chavez/Ryan): What/who is going to do the post processing. What does it look like and can we feedback/link to the results. ### Digital Camera (Jim Barry, Mark Chaffey): is still an unknown as to what will be returned to SSDS. Mark sent request to Jim B for requirements from the camera and that will help answer the questions for SSDS. ### CTD (SBE 16+) (Bill Ussler/Doug Conlin): Seabird software was identified as possibly doing the post processing, need to understand process in detail ### ADCP RDI Sentinel (Coenen): What is the post processing for this instrument? Can we feedback/link data products? ### Aquadopp (Ussler): Contact Leslie to get post processing software. ### Wetlabs ECO BB2F Triplett (Ussler): Follow up to try and get example data to figure out what next steps are ### Aanderaa Oxygen (Barry): Need to follow up to see what the data looks like and what post processing will be done. ### Nephalomter (Ussler): Get details on post processing. Is this a SIAM instrument? ### McLane Profiler(Hamilton): Get any details on post processing and if it is to link back to SSDS. ### Sediment Traps(Chavez): What will the data look like, are we going to put that in SSDS? ## General Issues: ## No decision has been made on MBARI's data policy for this experiment, currently the SSDS team is assuming all data will be open. The public access to data issue was discussed and the next step is to have the scientist's meet to come up with idea of how MSE data should be accessed by the public (if at all) and then give that feedback to the MT for review/approval (note the current default is that data that goes into SSDS is public) ## Documentation in spreadsheet incomplete -- we need to go to each of the reps and collect this data ### Don't know driver writers for many nominally SIAM instruments ## Not sure what the summary/snapshot data (ADCP, OCR, Nortek, Weblabs) will look like and if we are supposed to do anything with it. h3. Related Data Management Systems and Projects # [OpenIOOS.org|http://www.openioos.org/index.html] # [GEON|http://www.geongrid.org] # [Earth Systems Grid|https://www.earthsystemgrid.org] # [NOAA's OceanExplorer|http://oceanexplorer.noaa.gov/] # [GOMOOS|http://www.gomoos.org/] # [SeaMaven|http://www.seamaven.org/sm.swf] # [SEACOOS|http://seacoos.org/] # [The COOL Room|http://www.thecoolroom.org/] # [EARTH Observations|http://earthobservations.org/] # [Strategies.org|http://strategies.org/] # [DAPPER (Look at DAPPER to serve SSDS data)|http://www.epic.noaa.gov/epic/software/dapper/] # [PACOOS|http://www.pacoos.org/] # [CENCOOS|http://www.cencoos.org/] # [EGY|http://egy.org/] # [OceanSITES|http://www.oceansites.org/OceanSITES/] # [LDGO|http://ingrid.ldgo.columbia.edu/] # Christoph Waldman (University of Bremen, Germany) (presented at MBARI on Jan 9, 2006) ## Sensor interoperability and standardisation ## ESONET I and ESONET II + ESONIM, KM2NET, ANTARES, ORION ## He is working on standardized data quality and semantics ## Quality Control: important to always state accuracy ## Using SensorML to describe the sensor and data from sensor (but does not address semantics) ]]> 82 79 81 1179754 1212521 5 2007-06-22 13:07:08.573 2007-06-22 13:58:50.810 1179738 11273151 Administrative Tools->Internet Information Services, right-click on Default Web Site and select Properties. ## Then on the Home Directory tab, check the box next to Directory browsing. # I created the directory C:\Inetpub\wwwroot\ssdsdata where I will have SSDS store everything. With the data there, it will then be HTTP accessible. # I then browsed to http://jboss.org and downloaded JBoss 4.2.2GA and unzipped it to the C:\bin\jboss-4.2.2GA directory ## I started up JBoss from the command line and noticed that port 8080 had been taken by the SQL services. I shutdown all SQL services, started JBoss so it could claim those ports, then restarted the SQL components. # I then installed SmartSVN so that I could get the code from the Google SVN server. # I started up SmartSVN and added a repository for Google code: ## SVN Location: http://shore-side-data-system.googlecode.com/svn/trunk ## Read only username: shore-side-data-system-read-only # I checked the code out read-only to a directory with the name of the server (amphisbaena.usc.edu). # I installed the Flex 4 SDK. ## I downloaded the SDK as a zip file and then extracted it in the c:\bin directory ]]> 11240414 9797754 9830523 5 2009-04-14 22:56:40.067 2009-04-14 23:06:33.087 9797681 17236007 17268773 1 2011-05-30 20:39:26.043 2011-05-30 20:39:26.043 17236005 17236009 17268775 2 2011-05-30 20:39:26.043 2011-05-30 20:54:16.663 17236005 89 91 25035172 0 10355664 2014-02-18 00:39:11.150 2014-02-18 00:39:11.150 1179736 1212503 36 2006-10-17 21:39:33.643 2007-06-22 11:51:19.197 10 25035173 0 10355664 2014-02-18 00:44:45.603 2014-02-18 00:44:45.603 1179735 1212502 35 2006-10-17 21:39:33.643 2007-03-04 10:24:05.763 10 1179737 1212504 37 2006-10-17 21:39:33.643 2007-06-22 12:38:34.497 10 7766124 7798888 7 2007-08-03 07:17:05.643 2008-11-19 12:27:38.673 1179945 25362596 0 10355664 2014-02-27 16:43:31.050 2014-02-27 16:43:31.050 8388846 8356090 7766094 7798858 6 2006-10-24 21:49:36.650 2008-10-21 17:26:20.763 91 630 628 33 2006-10-17 21:39:33.643 2006-10-24 15:51:19.793 10 1179818 1212585 5 2006-10-24 15:12:15.247 2007-07-02 08:40:21.040 80 634 632 4 2007-01-17 10:11:40.717 2007-01-19 09:01:44.170 596 8388630 metadataRef # metadataRef->metadataSequenceNumber # metadataRef->dataDescriptionID | # metadataRef # metadataSequenceNumber # dataDescriptionID | | parentId | direct copy: # parentId->parentId # parentId->platformID | # parentId # platformID | | recordType | # MetadataPacket: recordType to 0 # Other: direct copy | recordType | | | # If MetadataPacket, packetType = 0 # If SensorDataPacket, packetType = 1 # If DeviceMessagePacket, packetType = 2 | packetType | | firstBufferLength | ignored | | | firstBuffer | First buffer depends on which type of packet # If MetadataPacket, copy "bytes" buffer # If SensorDataPacket, copy "dataBuffer" # If DeviceMessagePacket, copy "message" | firstBuffer | | secondBufferLength | ignored | | | secondBuffer | Only exists if MetadataPacket and will copy over "cause" buffer | secondBuffer | h6. DevicePacket to SIAM Byte Array This is done by the SIAM Exportable Packet class || DevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataRef | direct copy | metadataRef | | parentId | direct copy | parentId | | recordType | direct copy | recordType | | | This is set based on what type of packet: # If MetadataPacket, set to 0x0101 # If SensorDataPacket, set to 0x0102 # If DeviceMessagePacket, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | firstBufferLength | direct copy (see note for first buffer) | firstBufferLength | | firstBuffer | This depends on the type of packet: # If MetadataPacket, "cause" bytes are copied over # If SensorDataPacket, "dataMessage" bytes are copied over # If DeviceMessagePacket, "message" bytes are copied over | firstBuffer | | secondBufferLength | only valid with MetadataPacket, but is copied directly over | secondBufferLength | | secondBuffer | only valid with MetadataPacket, and the "buffer" bytes are copied over | secondBuffer | h6. SSDSDevicePacket to SIAM Byte Array *Originally done in TransmogrifyMDB by calling SSDSDevicePacket.convertToPublishableVersion3ByteArray before passing byte array to method to translate to SSDS format and then send to ingest* || SSDSDevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataSequenceNumber | direct copy | metadataRef | | parentID | direct copy | parentID | | recordType | # If packetType = 0, set recordType = 0 # If packetType = 1, set recordType = recordType # If packetType = 2, set recordType = recordType | recordType | | | This is set based on what type of packet: # If packetType = 0, set to 0x0101 # If packetType = 1, set to 0x0102 # If packetType = 2, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | | This depends on the type of packet: # If packetType = 0, set to length of "otherBuffer" # If packetType = 1, set to length of "dataBuffer" # If packetType = 2, set to length of "dataBuffer" | firstBufferLength | | other/dataBuffer | This depends on the type of packet: # If packetType = 0, set to bytes from "otherBuffer" # If packetType = 1, set to bytes from "dataBuffer" # If packetType = 2, set to bytes from "dataBuffer" | firstBuffer | | | This only exists if it is packetType = 0 then it is set to the length of the "dataBuffer" | secondBufferLength | | dataBuffer | This only exists if it is packetType = 0 then it is set to the byte from the "dataBuffer" | secondBuffer | h5. SSDSDevicePacket to SSDS Byte Array *Originally done in SSDSDevicePacket.convertToVersion3ByteArray* || SSDSDevicePacket || Translation Rule || SSDS Byte Array || | sourceID | direct copy | sourceID | | systemTime | ignored during the translation directly, but used through getter methods for seconds and nanoseconds | | | timestampSeconds | direct copy (note that this is _sort of_ a direct copy, there are getter methods on SSDSDevicePacket that convert the systemTime to seconds and nanoseconds when called). | timestampSeconds | | timestampNanoseconds | direct copy (note that this is _sort of_ a direct copy, there are getter methods on SSDSDevicePacket that convert the systemTime to seconds and nanoseconds when called). | timestampNanoseconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | ignored | | | parentID | ignored | | | recordType | If packetType = 0, set packetSubType to 0, otherwise set to recordType | packetSubType | | packetType | Depends on packetType: # If packetType = 0, set to 1 # If packetType = 1, set to 0 # If packetType = 2, set to 4 | packetType | | metadataSequenceNumber | direct copy | metadataSequenceNumber | | dataDescriptionVersion | direct copy | dataDescriptionVersion | | platformID | direct copy | parentID | | | copy length of dataBuffer | firstBufferLength | | dataBuffer | direct copy | firstBuffer | | | copy length of otherBuffer | secondBufferLength | | otherBuffer | direct copy | secondBuffer | h6. SIAM Byte Array to SSDS Byte Array *Originally done in TransmogrifyMDB in checkAndPublishBytes method* || SIAM Byte Array || Translation Rules || SSDS Byte Array || | EX_DEVICEPACKET | ignored | | | serialVersionUID | ignored | | | sourceID | direct copy | sourceID | | | Depending on EX_XXXXXXXPACKET: # If MetadataPacket, set packetType to 1 # If SensorDataPacket, set packetType to 0 # If DeviceMessagePacket, set packetType to 4 | packetType | | | This was set using the SIAMMetadataTracker that tried to keep track of real version numbers based on XML in payload | metadataSequencNumber | | systemTime | Split into timestampSeconds and timestampNanoseconds | # timestampSeconds # timestampNanoSeconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | direct copy | dataDescriptionVersion | | parentId | direct copy | parentID | | recordType | If MetadataPacket (determined from EX_XXXXXXXPACKET), recordType set to 0, otherwise set to recordType | packetSubType | | EX_XXXXXXXPACKET | ignored in storage, but used in logic | | | serialVersionUID | ignored | | | first/secondBufferLength | If MetadataPacket (determined from EX_XXXXXXXPACKET), firstBufferLength is set to secondBufferLength so we can flip the "cause" and "buffer" bytes because it just made more sense since the cause was rarely populated. Otherwise set to firstBufferLength | firstBufferLength | | first/secondBuffer | If MetadataPacket (determined from EX_XXXXXXXPACKET), firstBuffer is set to secondBuffer so we can flip the "cause" and "buffer" bytes because it just made more sense since the cause was rarely populated. Otherwise set to firstBuffer | firstBuffer | | firstBufferLength | If MetadataPacket (determined from EX_XXXXXXXPACKET), secondBufferLength is set to firstBufferLength to flip "cause" and "buffer" bytes | secondBufferLength | | firstBuffer | If MetadataPacket (determined from EX_XXXXXXXPACKET), secondBuffer is set to firstBuffer to flip "cause" and "buffer" bytes | secondBuffer | h5. Exploration of Upgrade of Ingest/Transmogrify to AMQP In an effort to allow non-Java clients to send data to SSDS in the form of messages and to upgrade the messaging system to a technology that is more scalable and higher performance, an investigation of AMQP implementations was done. # [Qpid Exploration] ]]> 8355863 633 631 3 2007-01-17 10:11:40.717 2007-01-19 08:56:06.213 596 1179811 1212578 9 2007-07-02 09:05:44.180 2007-07-08 23:00:46.433 1179794 632 630 2 2007-01-17 10:11:40.717 2007-01-19 08:43:12.027 596 7766095 7798859 7 2006-10-24 21:49:36.650 2008-11-14 09:59:45.760 91 631 629 1 2007-01-17 10:11:40.717 2007-01-17 10:11:40.717 596 9797691 9830460 1 2009-04-14 23:11:31.037 2009-04-14 23:11:31.037 9797689 1179808 1212575 6 2007-07-02 09:05:44.180 2007-07-03 14:12:38.833 1179794 11010507 81 2009-10-28 17:10:29.877 2009-10-28 17:10:29.877 10388616 Databases->SSDS_Data->Tables and you should see something like this: !Figure 1.jpg|align=centre! You are looking for the *DeviceQCPlotConfig* table and if you scroll down and right-click on it, then choose 'Open Table->Return all Rows' you will see the listing of all the configurations for creating plots (Figure 2). !Figure 2.jpg|align=centre! Here is a list of the columns and what they mean: * *DeviceID_FK*: This is the SSDS ID of the instrument whose data you want to plot. * *PacketSubType*: This is the RecordType that contains the variable you want to plot. Some instruments have more than one type of data they produce. In order to differentiate those, all device data descriptions must specify a RecordType (usually it is a 1, but not always). You can consult the SSDS web interface ([http://new-ssds.mbari.org]) or the device XML to find what the RecordTypes are. * *RecordVariable_Name*: This is the short name of the variable you want to plot. * *NumberOfHoursBack*: This tells the software how far back (in hours) you want to go to grab the data to plot. It will grab data back that many hours until the current time and create a plot of that data. * *Chart_Type*: * *X_Size*: * *Y_Size*: * *X_Axis_Label*: * *Y_Axis_Label*: * *Title*: * *Page_Title*: * *Page_URL_String*: * *Image_URL_String*: * *Data_Page_URL_String*: * *Y_Max*: * *Y_Min*: * *Y_Axis_One_Or_Two*: * *Use_Long_Variable_Name*: * *Gps_Show_Anchor*: * *Gps_Anchor_Latitude*: * *Gps_Show_Watch_Circle*: * *Gps_Watch_Circle_Diameter_Km*: * *Gps_Scale_Chart_To_Fit_Data*: * *GraphOn*: ]]> 10355890 11010506 81 2009-10-28 17:10:29.877 2009-10-28 17:10:29.877 1179810 1212577 8 2007-07-02 09:05:44.180 2007-07-08 22:28:49.043 1179794 11010505 81 2009-10-28 17:10:29.877 2009-10-28 17:10:29.877 7766099 7798863 8 2006-10-24 21:49:36.650 2008-11-14 10:00:11.237 91 1179809 1212576 7 2007-07-02 09:05:44.180 2007-07-08 16:20:27.810 1179794 11010504 81 2009-10-28 17:10:29.877 2009-10-28 17:10:29.877 1179804 1212571 5 2007-07-02 09:05:44.180 2007-07-03 13:32:04.027 1179794 1179803 1212570 4 2007-07-02 09:05:44.180 2007-07-03 11:28:30.210 1179794 15728688 80 2011-01-05 09:08:06.593 2011-01-05 09:08:06.593 108373 1 9797682 9830451 1 2009-04-14 22:56:40.067 2009-04-14 22:56:40.067 9797681 15728689 80 2011-01-05 09:27:04.053 2011-01-05 09:27:04.053 97682 1 1179801 1212568 2 2007-07-02 09:05:44.180 2007-07-02 09:54:49.493 1179794 9797679 9830448 1 2006-10-24 15:13:04.743 2006-10-24 15:13:04.743 81 1179802 1212569 3 2007-07-02 09:05:44.180 2007-07-03 11:27:02.483 1179794 7766113 7798877 9 2006-10-24 21:49:36.650 2008-11-14 11:05:34.133 91 1179795 1212562 1 2007-07-02 09:05:44.180 2007-07-02 09:05:44.180 1179794 15728692 80 2011-01-05 10:44:31.347 2011-01-05 10:44:31.347 129070 1 7766114 7798878 10 2006-10-24 21:49:36.650 2008-11-18 16:50:37.280 91 15728690 80 2011-01-05 10:42:59.017 2011-01-05 10:42:59.017 20143 1 15728691 80 2011-01-05 10:44:13.267 2011-01-05 10:44:13.267 363606 1 637 635 6 2007-01-17 10:11:40.717 2007-01-19 09:05:09.093 596 15728696 80 2011-01-05 15:16:06.717 2011-01-05 15:16:06.717 225348 1 7766118 7798882 4 2007-08-03 07:17:05.643 2007-11-16 11:51:05.710 1179945 638 636 1 2007-01-19 09:03:26.357 2007-01-19 09:03:26.357 635 15728697 80 2011-01-05 15:30:58.147 2011-01-05 15:30:58.147 546232 1 9797687 9830456 4 2009-04-14 22:56:40.067 2009-04-14 23:06:19.537 9797681 1179793 1212560 4 2006-10-24 15:12:15.247 2007-02-08 12:43:33.360 80 15728694 80 2011-01-05 13:04:41.600 2011-01-05 13:04:41.600 27461 1 7766116 7798880 11 2006-10-24 21:49:36.650 2008-11-18 16:51:09.757 91 636 634 5 2007-01-17 10:11:40.717 2007-01-19 09:02:51.210 596 15728695 80 2011-01-05 13:08:08.367 2011-01-05 13:08:08.367 150528 1 25035224 0 10355664 2014-02-18 19:32:26.217 2014-02-18 19:32:26.217 9797686 9830455 3 2009-04-14 22:56:40.067 2009-04-14 23:03:07.417 9797681 7766122 7798886 6 2007-08-03 07:17:05.643 2008-11-19 12:12:58.967 1179945 17268771 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat}\\ h4. B. Tips for debugging with Ferret To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the change log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is [http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log] # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: [http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED] To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:`PSAL,return=lend`/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC !-> LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:5182/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the first 3 temperature values from the 1m and 10m input CTD data and the first 3 times of the output axis: {noformat} yes? list/l=1:3/d=1 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1338 at original sampling intervals FILENAME : m1_ctd0001_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 1 122W 1 27-OCT-2010 21:08:43 / 1: 14.01 27-OCT-2010 21:22:27 / 2: 14.10 27-OCT-2010 21:28:23 / 3: 14.10 yes? list/l=1:3/d=2 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 27-OCT-2010 20:11:17 / 1: 13.38 27-OCT-2010 20:21:16 / 2: 13.59 27-OCT-2010 20:31:16 / 3: 13.45 yes? show/l=1:3 axis time name axis # pts start end TIME TIME 5181 r 27-OCT-2010 20:30 31-MAY-2011 16:30 T0 = 01-JAN-1950 00:00:00 Axis span (to cell edges) = 215.875 L T TBOX TBOXLO TSTEP (DAYS) 1> 27-OCT-2010 20:30:00 0.0416667 27-OCT-2010 20:00:00 22214.85 2> 27-OCT-2010 21:30:00 0.0416667 27-OCT-2010 21:00:00 22214.9 3> 27-OCT-2010 22:30:00 0.0416667 27-OCT-2010 22:00:00 22214.94 {noformat}\\ h4. C. Fixing the start times for all the child instrument deployments The problem seems to be that the 10m data begin at 20:11:17 before the 1m data at 21:08:43. The first write of the 1m data to the output netCDF file gives the file its shape with the KLIMITS= and LLIMITS= options. Writing the 10m data, which begins in before the bounds that were defined violates Ferret's axis ordering logic. To test this hypothesis edit the SAVE statement in the temporary .jnl file to make the start time for the 10m data 21:00, execute it and then examine the output file: {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 21:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc (Another ferret session) > ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 31-May-11 12:19 yes? use "/tmp/OS_MBARI-M1_20101027_R_TS.nc" yes? list/l=1:10 temp VARIABLE : Hourly sea_water_temperature (celsius) FILENAME : OS_MBARI-M1_20101027_R_TS.nc FILEPATH : /tmp/ SUBSET : 11 by 10 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 27-OCT-2010 21:30 / 1: 14.09 13.60 .... .... .... .... .... .... .... .... .... 27-OCT-2010 22:30 / 2: 14.17 13.62 .... .... .... .... .... .... .... .... .... 27-OCT-2010 23:30 / 3: 14.34 13.63 .... .... .... .... .... .... .... .... .... 28-OCT-2010 00:30 / 4: 14.28 13.69 .... .... .... .... .... .... .... .... .... 28-OCT-2010 01:30 / 5: 14.22 13.72 .... .... .... .... .... .... .... .... .... 28-OCT-2010 02:30 / 6: 14.21 13.78 .... .... .... .... .... .... .... .... .... 28-OCT-2010 03:30 / 7: 14.21 13.76 .... .... .... .... .... .... .... .... .... 28-OCT-2010 04:30 / 8: 14.09 13.79 .... .... .... .... .... .... .... .... .... 28-OCT-2010 05:30 / 9: 14.01 13.77 .... .... .... .... .... .... .... .... .... 28-OCT-2010 06:30 / 10: 13.89 13.76 .... .... .... .... .... .... .... .... .... yes? {noformat} O.K.\! That looks good. The 10m data got written. Now to fix it for production. This deployment was a special case where the 1m data start after the 10m data. Typically, it's most convenient to have all of the microcats have the same deployment start time. This makes the follow on data processing much simpler. All of the deployment start times in SSDS_METADTA for the M1 201010 deployment were adjusted to be the same as the start time for the Mooring deployment. Performing this step is now part of the [standard operating procedure for performing mooring turns|O3S:OASIS Mooring turn]. Once SSDS_METADATA has been updated the individual instrument netCDF files must be created by DStoNetCDF.pl before combineTS.pl in run. h4. D. Examining the gridding method There are still some problems with gappy data at 40m and below in the \_TS file. To debug these we'll make another copy of the OS_MBARI-M1_20101027_R_TS.jnl to /tmp and edit it to go through the processing of the 40m data saving the data to a netCDF file in /tmp so as not to disturb the production processing. We also comment out all of the "CANCEL DATA 1" statements so that we can examine data from the files. This file is executed and we see no errors. We need to examine the input data and the Ferret variables that are constructed for the gridding. Here are the commands from the .jnl that perform the the gridding for the 40m data: {noformat} ! ! Remove temperature outliers before gridding ! LET Temperature_QFLAG = IF Temperature GT 2 AND Temperature LT 20 THEN 1 ELSE (-99999) SET VAR/BAD=-99999 Temperature_QFLAG LET Temperature_QC = Temperature_QFLAG * Temperature ! ! Compute my own mean of the data, making sure to assign missing values in the gaps in the Gap FLAG ! Allow at least 1 data point in each destination cell - there are usually 6 for 10 minute data ! LET Temperature_GOOD = Temperature_QC[gt=TIME@SUM] / Temperature_QC[gt=TIME@NGD] LET Temperature_GFLAG = IF Temperature[gt=TIME@NGD,gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] LT 1 THEN (-99999) ELSE 1 SET VAR/BAD=-99999 Temperature_GFLAG LET TEMP = Temperature_GFLAG * Temperature_GOOD[gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] {noformat} Here are some sample Ferret commands for plotting these data from the session that just executed the OS_MBARI-M1_20101027_R_TS.jnl script: {noformat} yes? go "OS_MBARI-M1_20101027_R_TS.jnl" yes? set region/t=1-may-2011:1-jun-2011 yes? plot Temperature_QC yes? plot/ov/symbol=1 temperature yes? set win 2 yes? plot Temperature_QC[gt=TIME@NGD] yes? plot TEMPERATURE_QC[GT=TIME@SUM] {noformat} These last two plot commands indicate the source of the problem of the gappy data from the inductive modem microcats. Another big clue is the comment in the .jnl file referring to the 10-minute input data and having at least 6 good values within each gridding cell. As can be seen in the plot we often have less than 1 for the @NGD value for the 40m data. These data are hourly and we should be using a different gridding algorithm for them. !m1_ngd_sum.gif|thumbnail! The October 2010 M1 deployment was the first one where all the inductive modem connected CTDs were logged as separate instruments, and the first deployment where the serial microcat processing software was used to process the inductive modem microcat data. The fix for this problem appears to be to do a different gridding for the hourly data from 40m and below. This change was implemented in the code on 2 June 2011 using the Ferret same gridding transform (@MAX) that was previously used for the inductive modem data processing.  Here is the after figure: !wind_SEA_WATER_TEMPERATURE_HR_last30_after.gif|thumbnail! The subsurface data looks a lot better, but there are still some gaps.  If the gaps extend across the whole water column encompassing both the inductive modem and serial microcats then perhaps the gap is due to an interruption in data processing through oasisToSSDS.  This can be remedied by reprocessing the telemetered data by following the README instructions in /u/ssdsadmin/dev/DPforSSDS/oasis/scripts on elvis. h4. E. Testing the results of the gridding It is worthwhile to do another check on the quality of the gridding that is now being performed.  The following Ferret exercice will show the input data compared to the gridded data. {noformat} USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" SET REGION/T="29-May-2011 18:13":"02-Jun-2011 18:13" shade SEA_WATER_TEMPERATURE_HR set win 2 plot/k=1/vlimits=7:13 SEA_WATER_TEMPERATURE_HR repeat/k=2:11 plot/ov SEA_WATER_TEMPERATURE_HR {noformat} Produces these images showing missing data, especially from the 20m and 40m microcats: !m1_shade.gif|thumbnail! !m1_lines.gif|thumbnail! Let's compare the original instruments date from 20m and 40m to the gridded product, using the same Ferret session from above:" {noformat} use "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0020_20101027_original.nc" plot/symbol=1 temperature[d=2] plot/ov SEA_WATER_TEMPERATURE_HR[k=3,d=1] use "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0040_20101027_original.nc" plot/symbol=1 temperature[d=3] plot/ov SEA_WATER_TEMPERATURE_HR[k=4,d=1] {noformat} Gives: !m1_compare_20.gif|thumbnail! !m1_compare_40.gif|thumbnail! We obviously still have some gridding issues... h4. F. Rinse and repeat Now is the time to iterate using our method of starting with the .jnl file that aggregates and grids the data to the OceanSITES \_TS.nc file going back to step D. \\]]> 17236005 15728698 80 2011-01-05 15:31:19.680 2011-01-05 15:31:19.680 3736064 1 9797684 9830453 2 2009-04-14 22:56:40.067 2009-04-14 22:58:33.113 9797681 7766120 7798884 5 2007-08-03 07:17:05.643 2008-11-19 11:50:00.027 1179945 1179786 1212553 19 2007-06-22 13:07:08.573 2007-06-29 09:38:28.260 1179738 1179785 1212552 18 2007-06-22 13:07:08.573 2007-06-29 09:25:45.327 1179738 1179784 1212551 17 2007-06-22 13:07:08.573 2007-06-28 16:02:53.613 1179738 1179783 1212550 16 2007-06-22 13:07:08.573 2007-06-27 08:40:04.810 1179738 25559140 10355664 2014-03-05 15:53:41.237 2014-03-05 15:53:41.237 25362647 0 10355664 2014-02-28 21:35:03.633 2014-02-28 21:35:03.633 675 672 1 2007-01-22 14:36:00.070 2007-01-22 14:36:00.070 674 1179769 1212536 14 2007-06-22 13:07:08.573 2007-06-25 15:41:13.700 1179738 676 673 2 2007-01-22 14:36:00.070 2007-01-22 14:48:33.560 674 1179770 1212537 15 2007-06-22 13:07:08.573 2007-06-26 10:09:40.853 1179738 677 674 3 2007-01-22 14:36:00.070 2007-01-22 14:52:53.700 674 1179767 1212534 13 2007-06-22 13:07:08.573 2007-06-25 15:06:27.580 1179738 678 675 4 2007-01-22 14:36:00.070 2007-01-22 14:53:16.220 674 679 676 5 2007-01-22 14:36:00.070 2007-01-22 16:04:46.740 674 17236169 17268933 31 2011-05-30 20:39:26.043 2011-06-02 12:21:45.483 17236005 680 677 6 2007-01-22 14:36:00.070 2007-01-22 16:20:28.827 674 1179766 1212533 12 2007-06-22 13:07:08.573 2007-06-25 14:52:24.937 1179738 681 678 7 2007-01-22 14:36:00.070 2007-01-22 16:34:42.050 674 1179763 1212530 11 2007-06-22 13:07:08.573 2007-06-25 14:44:00.427 1179738 1179759 1212526 9 2007-06-22 13:07:08.573 2007-06-24 18:31:04.240 1179738 25559138 10355664 2014-03-05 15:53:41.237 2014-03-05 15:53:41.237 1179760 1212527 10 2007-06-22 13:07:08.573 2007-06-25 14:06:37.987 1179738 25559139 10355664 2014-03-05 15:53:41.237 2014-03-05 15:53:41.237 1179757 1212524 8 2007-06-22 13:07:08.573 2007-06-23 15:39:50.987 1179738 1179755 1212522 6 2007-06-22 13:07:08.573 2007-06-22 22:57:44.703 1179738 673 670 7 2007-01-17 10:11:40.717 2007-01-19 09:05:37.983 596 1179756 1212523 7 2007-06-22 13:07:08.573 2007-06-22 22:58:37.487 1179738 25362686 0 10355664 2014-03-01 12:26:14.290 2014-03-01 12:26:14.290 1 91 2006-10-24 21:59:58.580 2006-10-24 21:59:58.580 802702 1 16416861 16449624 71 2009-01-05 12:16:17.787 2011-03-18 09:11:54.210 8355863 16416860 16449623 70 2009-01-05 12:16:17.787 2011-03-18 09:10:18.367 8355863 17236108 17268872 29 2011-05-30 20:39:26.043 2011-06-02 12:09:00.183 17236005 17236110 17268874 30 2011-05-30 20:39:26.043 2011-06-02 12:16:41.393 17236005 11341548 841 2010-03-10 21:19:01.290 2010-03-10 21:19:01.290 11341549 841 2010-03-10 21:19:01.290 2010-03-10 21:19:01.290 25035301 0 10355664 2014-02-19 17:22:52.980 2014-02-19 17:22:52.980 17564847 11797071 2011-06-09 08:02:11.903 2011-06-09 08:02:11.903 11341540 841 2010-03-10 21:19:01.290 2010-03-10 21:19:01.290 11341541 841 2010-03-10 21:19:01.290 2010-03-10 21:19:01.290 16416858 16449621 69 2009-01-05 12:16:17.787 2011-02-04 16:54:08.397 8355863 11341542 841 2010-03-10 21:19:01.290 2010-03-10 21:19:01.290 25362696 0 10355664 2014-03-01 15:08:52.033 2014-03-01 15:08:52.033 11341543 841 2010-03-10 21:19:01.290 2010-03-10 21:19:01.290 11341544 841 2010-03-10 21:19:01.290 2010-03-10 21:19:01.290 11341545 841 2010-03-10 21:19:01.290 2010-03-10 21:19:01.290 11341546 841 2010-03-10 21:19:01.290 2010-03-10 21:19:01.290 11341547 841 2010-03-10 21:19:01.290 2010-03-10 21:19:01.290 25362720 0 10355664 2014-03-02 00:10:36.140 2014-03-02 00:10:36.140 17236089 17268854 24 2011-05-30 20:39:26.043 2011-06-02 09:46:06.803 17236005 11370504 1 10355664 2010-01-17 14:22:15.070 2010-01-17 14:22:15.070 17236087 17268852 23 2011-05-30 20:39:26.043 2011-06-02 00:22:00.307 17236005 17236085 17268850 22 2011-05-30 20:39:26.043 2011-06-02 00:09:14.227 17236005 17236083 17268848 21 2011-05-30 20:39:26.043 2011-06-01 23:56:17.740 17236005 17236081 17268846 20 2011-05-30 20:39:26.043 2011-06-01 23:45:23.557 17236005 17236079 17268844 19 2011-05-30 20:39:26.043 2011-06-01 16:19:54.193 17236005 1179829 1212596 10 2007-07-12 20:54:27.637 2007-07-12 22:57:11.780 1179819 10388734 New->Other...' and then select 'Server->Server'. Then 'Next>' # Then select 'JBoss->JBoss v4.2'. The default server host of 'localhost' and server name of 'JBoss v4.2 on localhost' is fine. Click on 'Next>'. # You can use the Default JRE and for the 'Application Server Directory' browse to the location where you unpacked JBoss. {warning:title=Under Construction} *_September 30, 2009_* this is still under construction {warning} ]]> 10356008 1179830 1212597 11 2007-07-12 20:54:27.637 2007-07-12 23:00:40.593 1179819 17236106 17268870 28 2011-05-30 20:39:26.043 2011-06-02 12:03:41.743 17236005 1179827 1212594 8 2007-07-12 20:54:27.637 2007-07-12 22:42:39.673 1179819 1179828 1212595 9 2007-07-12 20:54:27.637 2007-07-12 22:45:02.033 1179819 17236104 17268868 27 2011-05-30 20:39:26.043 2011-06-02 11:53:29.807 17236005 17236102 17268866 26 2011-05-30 20:39:26.043 2011-06-02 11:47:59.983 17236005 1179831 1212598 12 2007-07-12 20:54:27.637 2007-07-12 23:05:08.107 1179819 1179832 1212599 10 2007-07-02 09:05:44.180 2007-07-08 23:02:24.137 1179794 17236100 17268864 25 2011-05-30 20:39:26.043 2011-06-02 09:58:35.087 17236005 1179821 1212588 2 2007-07-12 20:54:27.637 2007-07-12 21:09:41.367 1179819 2831 635 2007-01-19 09:14:33.860 2007-01-19 09:14:33.860 1179822 1212589 3 2007-07-12 20:54:27.637 2007-07-12 21:39:09.287 1179819 1179820 1212587 1 2007-07-12 20:54:27.637 2007-07-12 20:54:27.637 1179819 1179825 1212592 6 2007-07-12 20:54:27.637 2007-07-12 22:22:59.827 1179819 1179826 1212593 7 2007-07-12 20:54:27.637 2007-07-12 22:27:04.350 1179819 1179823 1212590 4 2007-07-12 20:54:27.637 2007-07-12 21:51:05.240 1179819 1179824 1212591 5 2007-07-12 20:54:27.637 2007-07-12 22:22:21.593 1179819 2832 635 2007-01-19 09:14:33.860 2007-01-19 09:14:33.860 1179922 1212687 6 2006-10-24 15:12:15.247 2007-07-12 20:54:05.123 80 18777017 80 2012-02-03 09:16:05.930 2012-02-03 09:16:05.930 18777016 80 2012-02-03 09:16:05.930 2012-02-03 09:16:05.930 1179921 1212686 44 2007-06-22 13:07:08.573 2007-07-31 11:10:08.253 1179738 18777019 80 2012-02-03 09:16:05.930 2012-02-03 09:16:05.930 1179920 1212685 43 2007-06-22 13:07:08.573 2007-07-31 11:08:53.077 1179738 18777018 80 2012-02-03 09:16:05.930 2012-02-03 09:16:05.930 1179919 1212684 42 2007-06-22 13:07:08.573 2007-07-31 11:08:09.190 1179738 18777013 80 2012-02-03 09:16:05.930 2012-02-03 09:16:05.930 1179918 1212683 41 2007-06-22 13:07:08.573 2007-07-31 11:00:09.123 1179738 18777012 80 2012-02-03 09:16:05.930 2012-02-03 09:16:05.930 1179917 1212682 40 2007-06-22 13:07:08.573 2007-07-31 10:51:22.237 1179738 18777015 80 2012-02-03 09:16:05.930 2012-02-03 09:16:05.930 1179916 1212681 39 2007-06-22 13:07:08.573 2007-07-31 10:44:45.747 1179738 18777014 80 2012-02-03 09:16:05.930 2012-02-03 09:16:05.930 1179915 1212680 38 2007-06-22 13:07:08.573 2007-07-31 10:42:25.657 1179738 18777025 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777024 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777027 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777026 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777021 80 2012-02-03 09:16:05.930 2012-02-03 09:16:05.930 18777020 80 2012-02-03 09:16:05.930 2012-02-03 09:16:05.930 18777023 80 2012-02-03 09:16:05.930 2012-02-03 09:16:05.930 18777022 80 2012-02-03 09:16:05.930 2012-02-03 09:16:05.930 18777032 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777033 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 17465507 17498273 11 2010-04-27 15:25:51.250 2011-06-09 07:22:14.097 11797071 18777034 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777035 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 17465509 17498275 12 2010-04-27 15:25:51.250 2011-06-09 07:25:46.023 11797071 18777028 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777029 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777030 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777031 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777040 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777041 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1179946 1212711 1 2007-08-03 07:17:05.643 2007-08-03 07:17:05.643 1179945 17465499 17498265 8 2010-04-27 15:25:51.250 2011-04-14 15:14:35.417 11797071 18777042 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1179944 1212709 7 2006-10-24 15:12:15.247 2007-07-31 11:40:05.770 80 18777043 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777036 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 17465504 17498270 9 2010-04-27 15:25:51.250 2011-06-08 21:23:31.067 11797071 18777037 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777038 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 17465506 17498272 10 2010-04-27 15:25:51.250 2011-06-09 07:13:36.267 11797071 18777039 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777051 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777050 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777049 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777048 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777047 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777046 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777045 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777044 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777059 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 25035332 0 10355664 2014-02-19 20:18:52.013 2014-02-19 20:18:52.013 18777058 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777057 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1179898 1212663 21 2007-06-22 13:07:08.573 2007-07-31 09:13:46.880 1179738 18777056 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1179897 1212662 20 2007-06-22 13:07:08.573 2007-06-29 09:45:50.690 1179738 18777055 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777054 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777053 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777052 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 18777066 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1179903 1212668 26 2007-06-22 13:07:08.573 2007-07-31 09:27:13.227 1179738 18777067 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1179904 1212669 27 2007-06-22 13:07:08.573 2007-07-31 09:28:38.697 1179738 18777064 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1179905 1212670 28 2007-06-22 13:07:08.573 2007-07-31 09:35:00.253 1179738 18777065 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1179906 1212671 29 2007-06-22 13:07:08.573 2007-07-31 09:46:59.240 1179738 18777062 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1179899 1212664 22 2007-06-22 13:07:08.573 2007-07-31 09:14:08.293 1179738 18777063 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1179900 1212665 23 2007-06-22 13:07:08.573 2007-07-31 09:21:39.387 1179738 18777060 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1179901 1212666 24 2007-06-22 13:07:08.573 2007-07-31 09:22:37.500 1179738 18777061 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1179902 1212667 25 2007-06-22 13:07:08.573 2007-07-31 09:25:36.287 1179738 18777074 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1179911 1212676 34 2007-06-22 13:07:08.573 2007-07-31 10:17:26.760 1179738 18777075 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1179912 1212677 35 2007-06-22 13:07:08.573 2007-07-31 10:23:38.900 1179738 18777072 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1179913 1212678 36 2007-06-22 13:07:08.573 2007-07-31 10:34:29.257 1179738 18777073 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1179914 1212679 37 2007-06-22 13:07:08.573 2007-07-31 10:35:50.347 1179738 18777070 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1179907 1212672 30 2007-06-22 13:07:08.573 2007-07-31 09:51:45.923 1179738 18777071 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1179908 1212673 31 2007-06-22 13:07:08.573 2007-07-31 09:59:23.900 1179738 18777068 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1179909 1212674 32 2007-06-22 13:07:08.573 2007-07-31 10:09:00.180 1179738 18777069 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1179910 1212675 33 2007-06-22 13:07:08.573 2007-07-31 10:09:59.160 1179738 840 837 3 2006-10-24 15:12:15.247 2006-10-24 21:42:55.053 80 3670596 3637846 8061034 8093796 29 2006-10-24 15:12:15.247 2008-06-05 12:55:52.550 80 1179947 1212712 2 2007-08-03 07:17:05.643 2007-08-03 08:30:41.280 1179945 1179951 1212716 8 2006-10-24 15:12:15.247 2007-08-03 06:42:23.637 80 1179953 1212718 9 2006-10-24 15:12:15.247 2007-08-06 10:27:47.947 80 1179955 1212720 10 2006-10-24 15:12:15.247 2007-08-06 10:31:35.570 80 1179958 1212723 11 2006-10-24 15:12:15.247 2007-08-06 10:33:03.210 80 1179960 1212725 13 2006-10-24 15:12:15.247 2007-08-06 10:37:27.077 80 1179959 1212724 12 2006-10-24 15:12:15.247 2007-08-06 10:37:06.903 80 1179961 1212726 14 2006-10-24 15:12:15.247 2007-08-06 10:38:01.030 80 849 846 3 2007-02-08 12:50:50.907 2007-02-08 16:53:29.560 841 850 847 4 2007-02-08 12:50:50.907 2007-02-09 09:28:02.100 841 847 844 2 2007-02-08 12:50:50.907 2007-02-08 15:37:47.087 841 844 841 1 2007-02-08 12:50:50.907 2007-02-08 12:50:50.907 841 18777006 17236005 2012-02-03 09:16:05.853 2012-02-03 09:16:05.853 18777007 17236005 2012-02-03 09:16:05.853 2012-02-03 09:16:05.853 1179974 1212738 15 2006-10-24 15:12:15.247 2007-08-06 10:56:29.163 80 1179975 1212739 16 2006-10-24 15:12:15.247 2007-08-10 10:32:34.560 80 18777010 17236005 2012-02-03 09:16:05.853 2012-02-03 09:16:05.853 18777011 17236005 2012-02-03 09:16:05.853 2012-02-03 09:16:05.853 851 848 5 2007-02-08 12:50:50.907 2007-02-09 09:40:49.467 841 18777008 17236005 2012-02-03 09:16:05.853 2012-02-03 09:16:05.853 18777009 17236005 2012-02-03 09:16:05.853 2012-02-03 09:16:05.853 8061069 8093832 17 2008-12-18 11:24:49.237 2008-12-18 16:24:49.327 8061037 10455990 1179945 2009-06-16 10:34:34.290 2009-06-16 10:34:34.290 8061067 8093830 16 2008-12-18 11:24:49.237 2008-12-18 16:21:29.827 8061037 10455991 1179945 2009-06-16 10:34:34.290 2009-06-16 10:34:34.290 3670672 15 {noformat} to {noformat} 1 {noformat} Which should effectively make the RuminateMDB a singleton which should alleviate our deadlock issues that we were having (at least at Ruminate step). {note}]]> 3637922 8388898 8356145 8061044 8093807 3 2008-12-18 11:24:49.237 2008-12-18 12:49:23.620 8061037 8061046 8093809 4 2008-12-18 11:24:49.237 2008-12-18 12:54:42.750 8061037 8061048 8093811 5 2008-12-18 11:24:49.237 2008-12-18 12:55:48.500 8061037 8061050 8093813 7 2008-12-18 11:24:49.237 2008-12-18 12:58:47.893 8061037 8061049 8093812 6 2008-12-18 11:24:49.237 2008-12-18 12:58:22.190 8061037 8061036 8093798 30 2006-10-24 15:12:15.247 2008-12-18 09:52:46.867 80 8061038 8093801 31 2006-10-24 15:12:15.247 2008-12-18 11:23:10.637 80 8061040 8093803 1 2008-12-18 11:24:49.237 2008-12-18 11:24:49.237 8061037 8061042 8093805 2 2008-12-18 11:24:49.237 2008-12-18 11:49:10.303 8061037 8061059 8093822 12 2008-12-18 11:24:49.237 2008-12-18 13:21:02.477 8061037 8061061 8093824 13 2008-12-18 11:24:49.237 2008-12-18 13:46:35.360 8061037 8061063 8093826 14 2008-12-18 11:24:49.237 2008-12-18 13:48:16.837 8061037 8061065 8093828 15 2008-12-18 11:24:49.237 2008-12-18 14:38:53.620 8061037 8061052 8093815 8 2008-12-18 11:24:49.237 2008-12-18 12:59:21.600 8061037 8061053 8093816 9 2008-12-18 11:24:49.237 2008-12-18 13:02:04.797 8061037 4555174 4456506 2008-05-30 15:57:40.900 2008-05-30 15:57:40.900 4555175 4456506 2008-05-30 15:57:40.900 2008-05-30 15:57:40.900 8061055 8093818 10 2008-12-18 11:24:49.237 2008-12-18 13:02:34.200 8061037 4555176 4456506 2008-05-30 15:57:40.900 2008-05-30 15:57:40.900 8061057 8093820 11 2008-12-18 11:24:49.237 2008-12-18 13:13:12.343 8061037 25035463 0 10355664 2014-02-20 19:18:49.553 2014-02-20 19:18:49.553 13664455 13697206 10 2010-06-29 09:39:57.047 2010-06-29 12:55:48.177 13664436 13664458 13697209 12 2010-06-29 09:39:57.047 2010-06-29 15:55:38.070 13664436 13664457 13697208 11 2010-06-29 09:39:57.047 2010-06-29 13:00:18.180 13664436 13140041 13172807 8 2010-03-10 21:41:49.343 2010-03-10 22:31:07.937 11240414 13664452 13697203 8 2010-06-29 09:39:57.047 2010-06-29 12:45:27.993 13664436 13664453 13697204 9 2010-06-29 09:39:57.047 2010-06-29 12:48:58.703 13664436 13664448 13697199 6 2010-06-29 09:39:57.047 2010-06-29 11:48:04.023 13664436 13664450 13697201 7 2010-06-29 09:39:57.047 2010-06-29 12:42:39.087 13664436 25362438 0 10355664 2014-02-25 10:33:37.717 2014-02-25 10:33:37.717 13664444 13697195 4 2010-06-29 09:39:57.047 2010-06-29 11:10:49.670 13664436 13664446 13697197 5 2010-06-29 09:39:57.047 2010-06-29 11:24:23.533 13664436 13664440 13697191 2 2010-06-29 09:39:57.047 2010-06-29 10:01:32.777 13664436 13664442 13697193 3 2010-06-29 09:39:57.047 2010-06-29 10:50:27.340 13664436 25035572 0 10355664 2014-02-23 03:24:20.880 2014-02-23 03:24:20.880 13664438 13697189 1 2010-06-29 09:39:57.047 2010-06-29 09:39:57.047 13664436 13664434 13697185 50 2006-10-24 15:12:15.247 2010-04-27 15:57:15.503 80 13860906 80 2010-07-01 06:19:32.023 2010-07-01 06:19:32.023 32256 1 13664489 13697240 15 2010-06-29 09:39:57.047 2010-06-30 09:44:03.207 13664436 25035511 0 10355664 2014-02-21 07:44:53.650 2014-02-21 07:44:53.650 13140051 13172817 3 2009-01-12 13:08:50.467 2010-01-04 16:18:47.140 8356090 13140045 13172811 10 2010-03-10 21:41:49.343 2010-06-03 17:00:17.540 11240414 13664462 13697213 14 2010-06-29 09:39:57.047 2010-06-30 08:06:10.560 13664436 13140043 13172809 9 2010-03-10 21:41:49.343 2010-06-03 16:46:26.960 11240414 13664460 13697211 13 2010-06-29 09:39:57.047 2010-06-29 15:56:51.497 13664436 25362938 0 10355664 2014-03-04 17:44:20.993 2014-03-04 17:44:20.993 9928756 81 2009-05-06 08:24:26.333 2009-05-06 08:24:26.333 3483 1 9928757 81 2009-05-06 08:24:46.723 2009-05-06 08:24:46.723 19851 1 10461972 10356413 2009-10-07 22:55:32.257 2009-10-07 22:55:32.257 25362957 0 10355664 2014-03-05 00:40:44.370 2014-03-05 00:40:44.370 10461973 10356413 2009-10-07 22:55:32.257 2009-10-07 22:55:32.257 8355892 8388659 33 2006-10-24 15:12:15.247 2009-01-05 12:15:09.700 80 633 (is this correct?) **** Andrew: 20 days # Need project schedule for 2006(this week) # Development ** General *** New machine order HP dual athalon with Red Hat **** Need to draft up deployment and migration scenario for new architecture/new machine *** Web Application **** Sputtering to life ([http://ssdsdevpc:8080/ssds]) *** Core Metadata *** Core Data *** Clients *** Model Integration # MSE *** Need to document requirements for data access (by tomorrow!) *** No update on data policy # ASAP *** Need to meet with Mike G to discuss schema changes. h2. Action Items # Getting new architecture rolled out is critical (time wise) # Finish up DTS job migration # Get project plan done and identify tasks for developers # Look at [eventbus:|https://eventbus.dev.java.net/] as possible solution for remote model object notificaton (and other SSDS stuff). # Finish getting John's development environment going # Get ASAP update from Mike G. ]]> 635 8355896 8388661 34 2006-10-24 15:12:15.247 2009-01-07 14:15:10.550 80 8355885 8388652 2 2009-01-05 12:16:17.787 2009-01-05 12:16:39.707 8355863 5505494 5374184 2008-06-17 16:50:54.140 2008-06-17 16:50:54.140 2162890 2195656 46 2007-06-22 13:07:08.573 2007-11-14 00:23:32.773 1179738 2162889 2195655 45 2007-06-22 13:07:08.573 2007-07-31 11:22:17.230 1179738 8913100 8945860 21 2007-02-08 12:50:50.907 2009-01-23 11:06:33.383 841 2162893 2195659 49 2007-06-22 13:07:08.573 2007-11-14 06:45:14.453 1179738 2162891 2195657 47 2007-06-22 13:07:08.573 2007-11-14 00:24:13.560 1179738 2162892 2195658 48 2007-06-22 13:07:08.573 2007-11-14 06:43:53.280 1179738 594 596 10461971 10356413 2009-10-07 22:55:32.257 2009-10-07 22:55:32.257 10461970 10356413 2009-10-07 22:55:32.257 2009-10-07 22:55:32.257 10461969 10356413 2009-10-07 22:55:32.257 2009-10-07 22:55:32.257 10461968 10356413 2009-10-07 22:55:32.257 2009-10-07 22:55:32.257 9928709 9797681 2009-04-14 23:02:09.957 2009-04-14 23:02:09.957 48640 1 25035000 0 10355664 2014-02-16 00:10:42.493 2014-02-16 00:10:42.493 9928708 9797681 2009-04-14 22:57:42.137 2009-04-14 22:57:42.137 37376 1 595 597 9928713 81 2009-04-17 08:25:33.317 2009-04-17 08:25:33.317 40448 1 25035010 0 10355664 2014-02-16 02:55:20.547 2014-02-16 02:55:20.547 10355678 10388406 6 2009-07-31 11:41:34.023 2009-08-01 08:55:08.283 10355664 25034983 0 10355664 2014-02-15 17:19:05.567 2014-02-15 17:19:05.567 10355676 10388404 5 2009-07-31 11:41:34.023 2009-08-01 00:47:38.683 10355664 8913010 8945773 13 2007-02-08 12:50:50.907 2009-01-22 14:26:13.797 841 10355682 10388410 8 2009-07-31 11:41:34.023 2009-08-01 09:18:21.800 10355664 23953636 0 10355664 2014-01-24 07:12:10.790 2014-01-24 07:12:10.790 10355680 10388408 7 2009-07-31 11:41:34.023 2009-08-01 09:05:00.567 10355664 8913006 8945769 12 2007-02-08 12:50:50.907 2008-08-13 13:01:28.710 841 5406932 5374184 8913016 8945779 16 2007-02-08 12:50:50.907 2009-01-23 09:17:27.130 841 10355684 10388412 9 2009-07-31 11:41:34.023 2009-08-01 10:28:32.150 10355664 8913017 8945780 17 2007-02-08 12:50:50.907 2009-01-23 09:22:32.020 841 10355689 10388417 11 2009-07-31 11:41:34.023 2009-08-03 00:36:56.900 10355664 8913012 8945775 14 2007-02-08 12:50:50.907 2009-01-23 08:41:28.837 841 10355687 10388415 10 2009-07-31 11:41:34.023 2009-08-01 10:33:39.123 10355664 8913014 8945777 15 2007-02-08 12:50:50.907 2009-01-23 09:16:23.360 841 8913019 8945782 18 2007-02-08 12:50:50.907 2009-01-23 09:23:30.033 841 10355670 10388398 2 2009-07-31 11:41:34.023 2009-07-31 17:09:55.113 10355664 23953641 0 10355664 2014-01-25 06:11:39.347 2014-01-25 06:11:39.347 10355668 10388396 1 2009-07-31 11:41:34.023 2009-07-31 11:41:34.023 10355664 10355674 10388402 4 2009-07-31 11:41:34.023 2009-07-31 17:34:09.687 10355664 8913027 8945790 19 2007-02-08 12:50:50.907 2009-01-23 09:26:18.490 841 10355672 10388400 10322085 3 2009-07-31 11:41:34.023 2009-07-31 17:21:25.367 10355664 23953643 0 10355664 2014-01-25 18:02:53.733 2014-01-25 18:02:53.733 8913029 8945792 20 2007-02-08 12:50:50.907 2009-01-23 11:04:21.910 841 2162835 2195601 17 2006-10-24 15:12:15.247 2007-08-10 10:34:35.710 80 25362913 0 10355664 2014-03-04 10:59:52.353 2014-03-04 10:59:52.353 2162836 2195602 18 2006-10-24 15:12:15.247 2007-10-29 10:50:52.960 80 25362915 0 10355664 2014-03-04 11:02:06.210 2014-03-04 11:02:06.210 8355864 8388631 1 2009-01-05 12:16:17.787 2009-01-05 12:16:17.787 8355863 6226616 6259415 8355860 8388627 14 2006-10-24 21:49:36.650 2008-12-30 13:29:30.727 91 8355859 8388626 13 2006-10-24 21:49:36.650 2008-12-30 13:12:33.063 91 8355861 8388628 32 2006-10-24 15:12:15.247 2008-12-18 11:25:42.127 80 8355857 8388624 12 2006-10-24 21:49:36.650 2008-11-18 17:14:54.440 91 23953610 0 10355664 2014-01-21 09:58:15.153 2014-01-21 09:58:15.153 3114484 3179956 19 2006-10-24 15:12:15.247 2007-10-29 10:52:13.487 80 25034947 0 10355664 2014-02-15 03:02:28.240 2014-02-20 03:46:03.290 25362820 0 10355664 2014-03-03 11:00:30.913 2014-03-03 11:00:30.913 25362816 0 10355664 2014-03-03 08:50:44.833 2014-03-03 08:50:44.833 25362815 0 10355664 2014-03-03 08:30:28.017 2014-03-03 08:30:28.017 5013716 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 92 90 1 2006-10-24 21:49:36.650 2006-10-24 21:49:36.650 91 25362813 0 10355664 2014-03-03 08:08:08.560 2014-03-03 08:08:08.560 23953573 0 10355664 2014-01-16 00:55:27.413 2014-01-16 00:55:27.413 5013717 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 1540129 80 2007-07-31 11:38:47.267 2007-07-31 11:38:47.267 15360 1 5013718 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 94 92 3 2006-10-24 21:49:36.650 2006-10-24 21:55:57.813 91 5013719 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 93 91 2 2006-10-24 21:49:36.650 2006-10-24 21:55:15.150 91 5013720 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 25362809 0 10355664 2014-03-03 07:04:22.133 2014-03-03 07:04:22.133 5013721 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 5013722 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 5013723 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 83 81 1 2006-10-24 15:18:56.927 2006-10-24 15:18:56.927 82 8356013 8388774 16 2009-01-05 12:16:17.787 2009-01-09 21:32:04.073 8355863 84 82 2 2006-10-24 15:18:56.927 2006-10-24 15:25:15.973 82 85 83 3 2006-10-24 15:18:56.927 2006-10-24 15:32:54.277 82 86 84 30 2006-10-17 21:39:33.643 2006-10-24 15:05:39.850 10 8356012 8388773 15 2009-01-05 12:16:17.787 2009-01-09 21:26:20.673 8355863 87 85 31 2006-10-17 21:39:33.643 2006-10-24 15:43:49.193 10 8356017 8388778 18 2009-01-05 12:16:17.787 2009-01-09 22:23:00.147 8355863 88 86 32 2006-10-17 21:39:33.643 2006-10-24 15:50:34.460 10 8356018 8388779 36 2006-10-24 15:12:15.247 2009-01-08 11:37:49.700 80 23953576 0 10355664 2014-01-18 00:41:01.147 2014-01-18 00:41:01.147 89 87 1 2006-10-24 15:12:15.247 2006-10-24 15:12:15.247 80 23953575 0 10355664 2014-01-18 00:05:47.887 2014-01-18 00:05:47.887 8356015 8388776 17 2009-01-05 12:16:17.787 2009-01-09 21:41:56.140 8355863 90 88 2 2006-10-24 15:12:15.247 2006-10-24 21:42:02.113 80 25034912 0 10355664 2014-02-14 12:59:30.450 2014-02-14 12:59:30.450 23953574 0 10355664 2014-01-16 16:51:44.137 2014-01-16 16:51:44.137 75 73 25 2006-10-17 21:39:33.643 2006-10-18 18:54:24.920 10 76 74 26 2006-10-17 21:39:33.643 2006-10-24 14:27:00.763 10 77 75 27 2006-10-17 21:39:33.643 2006-10-24 14:27:59.163 10 25034909 0 10355664 2014-02-14 11:44:03.380 2014-02-14 11:44:03.380 78 76 28 2006-10-17 21:39:33.643 2006-10-24 14:29:12.727 10 8356020 8388781 37 2006-10-24 15:12:15.247 2009-01-10 19:51:29.610 80 25034908 0 10355664 2014-02-14 10:22:56.030 2014-02-14 10:22:56.030 79 77 29 2006-10-17 21:39:33.643 2006-10-24 14:44:48.867 10 23953584 0 10355664 2014-01-18 23:44:46.330 2014-01-18 23:44:46.330 25034905 0 10355664 2014-02-14 09:46:52.250 2014-02-14 09:46:52.250 8356060 8388819 21 2009-01-05 12:16:17.787 2009-01-12 10:46:50.527 8355863 8912981 8945745 2 2009-01-22 10:35:49.373 2009-01-22 10:36:38.853 8912969 8356062 8388821 22 2009-01-05 12:16:17.787 2009-01-12 11:03:29.790 8355863 8912979 8945743 16 2009-01-08 16:16:54.793 2009-01-22 10:38:47.420 8355981 8356064 8388823 23 2009-01-05 12:16:17.787 2009-01-12 11:08:42.127 8355863 8912985 8945749 3 2009-01-22 10:35:49.373 2009-01-22 10:40:23.453 8912969 5013699 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 8356065 8388824 24 2009-01-05 12:16:17.787 2009-01-12 11:12:46.280 8355863 8912983 8945747 17 2009-01-08 16:16:54.793 2009-01-22 10:39:04.507 8355981 8356068 8388827 25 2009-01-05 12:16:17.787 2009-01-12 11:13:21.357 8355863 62 60 22 2006-10-17 21:39:33.643 2006-10-18 16:34:58.057 10 8912974 8945738 13 2009-01-08 16:16:54.793 2009-01-22 10:36:28.997 8355981 8912973 8945737 1 2009-01-22 10:35:49.373 2009-01-22 10:35:49.373 8912969 61 59 21 2006-10-17 21:39:33.643 2006-10-18 16:17:16.570 10 8356070 8388829 26 2009-01-05 12:16:17.787 2009-01-12 11:23:58.610 8355863 60 58 20 2006-10-17 21:39:33.643 2006-10-18 16:13:06.350 10 8912972 8945736 12 2009-01-08 16:16:54.793 2009-01-21 14:18:00.547 8355981 8356072 8388831 27 2009-01-05 12:16:17.787 2009-01-12 11:29:56.313 8355863 8912978 8945742 15 2009-01-08 16:16:54.793 2009-01-22 10:38:25.297 8355981 9798261 9831014 3 2006-10-24 15:13:04.743 2009-05-06 08:23:54.810 81 8912977 8945741 14 2009-01-08 16:16:54.793 2009-01-22 10:37:05.430 8355981 9798260 9831013 2 2006-10-24 15:13:04.743 2009-04-14 22:55:24.080 81 64 62 24 2006-10-17 21:39:33.643 2006-10-18 18:54:10.637 10 8912975 8945739 1 2009-01-22 10:36:03.900 2009-01-22 10:36:03.900 8912971 63 61 23 2006-10-17 21:39:33.643 2006-10-18 16:37:27.920 10 5013711 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 18777082 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 5013710 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 18777083 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 5013709 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 18777080 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 1540118 1179819 2007-07-13 00:06:28.787 2007-07-13 00:06:28.787 5211648 1 5013708 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 18777081 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 15630761 15663518 58 2006-10-24 15:12:15.247 2011-01-05 09:27:33.563 80 5013715 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 18777078 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 57 55 18 2006-10-17 21:39:33.643 2006-10-18 14:08:21.937 10 5013714 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 18777079 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 15630755 15663512 55 2006-10-24 15:12:15.247 2011-01-05 09:08:53.823 80 58 56 19 2006-10-17 21:39:33.643 2006-10-18 15:15:26.023 10 5013713 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 18777076 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 15630758 15663515 57 2006-10-24 15:12:15.247 2011-01-05 09:11:12.517 80 5013712 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 18777077 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 15630757 15663514 56 2006-10-24 15:12:15.247 2011-01-05 09:09:59.457 80 5013703 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 10355440 10388171 38 2007-02-08 12:50:50.907 2009-07-16 17:53:13.377 841 5013702 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 5013701 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 8356053 8388812 19 2009-01-05 12:16:17.787 2009-01-09 22:26:43.720 8355863 1540110 1179794 2007-07-08 23:06:12.750 2007-07-08 23:06:12.750 30720 1 18777088 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 43 41 16 2006-10-17 21:39:33.643 2006-10-18 14:07:24.237 10 5013700 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 18777089 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 15630753 15663510 54 2006-10-24 15:12:15.247 2010-11-08 10:38:32.500 80 44 42 17 2006-10-17 21:39:33.643 2006-10-18 14:07:38.577 10 5013707 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 8356055 8388814 20 2009-01-05 12:16:17.787 2009-01-12 10:39:19.290 8355863 10355436 10388167 36 2007-02-08 12:50:50.907 2009-07-16 17:17:13.980 841 18777086 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 5013706 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 18777087 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 5013705 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 10355438 10388169 37 2007-02-08 12:50:50.907 2009-07-16 17:25:03.067 841 1540114 1179794 2007-07-09 13:05:07.270 2007-07-09 13:05:47.520 31744 2 18777084 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 5013704 3637922 2008-06-09 09:36:21.913 2008-06-09 09:36:21.913 1540115 1179794 2007-07-09 13:05:07.270 2007-07-09 13:05:07.270 31744 1 1540114 18777085 80 2012-02-03 09:16:05.933 2012-02-03 09:16:05.933 40 38 14 2006-10-17 21:39:33.643 2006-10-18 14:05:46.927 10 39 37 13 2006-10-17 21:39:33.643 2006-10-18 14:04:03.480 10 41 39 15 2006-10-17 21:39:33.643 2006-10-18 14:06:58.947 10 36 34 12 2006-10-17 21:39:33.643 2006-10-18 14:02:01.120 10 35 33 11 2006-10-17 21:39:33.643 2006-10-18 13:54:11.407 10 15630775 15663532 60 2006-10-24 15:12:15.247 2011-01-05 10:45:21.417 80 23953504 0 10355664 2014-01-13 11:02:37.427 2014-01-13 11:02:37.427 32 30 9 2006-10-17 21:39:33.643 2006-10-18 13:52:27.157 10 31 29 8 2006-10-17 21:39:33.643 2006-10-18 13:50:06.967 10 15630763 15663520 59 2006-10-24 15:12:15.247 2011-01-05 10:43:30.713 80 34 32 10 2006-10-17 21:39:33.643 2006-10-18 13:52:44.777 10 28 26 7 2006-10-17 21:39:33.643 2006-10-18 13:49:26.923 10 27 25 6 2006-10-17 21:39:33.643 2006-10-18 13:46:54.087 10 8355953 8388717 35 2006-10-24 15:12:15.247 2009-01-07 14:16:57.980 80 25 23 5 2006-10-17 21:39:33.643 2006-10-17 23:51:33.733 10 15 13 2 2006-10-17 21:39:33.643 2006-10-17 23:19:16.707 10 15630781 15663538 62 2006-10-24 15:12:15.247 2011-01-05 13:06:50.507 80 16 14 3 2006-10-17 21:39:33.643 2006-10-17 23:44:45.700 10 17 15 4 2006-10-17 21:39:33.643 2006-10-17 23:45:01.610 10 15630779 15663536 61 2006-10-24 15:12:15.247 2011-01-05 13:04:14.027 80 25362765 0 10355664 2014-03-02 14:05:14.930 2014-03-02 14:05:14.930 23953524 0 10355664 2014-01-14 01:39:06.730 2014-01-14 01:39:06.730 5046287 3637922 2008-05-16 13:41:42.110 2008-05-16 13:41:42.110 313667 1 3735622 14 12 1 2006-10-17 21:39:33.643 2006-10-17 21:39:33.643 10 13434989 91 2010-06-09 08:49:22.777 2010-06-09 08:49:22.777 23953462 0 10355664 2013-12-30 10:41:44.657 2013-12-30 10:41:44.657 15630803 15663560 66 2006-10-24 15:12:15.247 2011-01-05 15:20:04.820 80 15706105 10 2011-01-05 15:20:04.813 2011-01-05 15:20:04.813 8912922 8945689 10 2009-01-08 16:16:54.793 2009-01-15 12:54:52.620 8355981 13434988 91 2010-06-09 08:49:22.777 2010-06-09 08:49:22.777 8355999 8388760 7 2009-01-05 12:16:17.787 2009-01-09 13:51:15.420 8355863 22020097 519 10355664 2013-09-02 16:29:15.080 2014-03-05 14:26:15.030 23953463 0 10355664 2013-12-30 21:33:47.133 2013-12-30 21:33:47.133 15630804 15663561 67 2006-10-24 15:12:15.247 2011-01-05 15:22:30.790 80 15706104 10 2011-01-05 15:20:04.813 2011-01-05 15:20:04.813 23953464 0 10355664 2013-12-31 20:42:07.960 2013-12-31 20:42:07.960 15630805 15663562 68 2006-10-24 15:12:15.247 2011-01-05 15:22:49.203 80 15706107 10 2011-01-05 15:20:04.813 2011-01-05 15:20:04.813 8356001 8388762 8 2009-01-05 12:16:17.787 2009-01-09 14:19:17.147 8355863 23953465 0 10355664 2014-01-01 07:21:19.433 2014-01-01 07:21:19.433 15706106 10 2011-01-05 15:20:04.813 2011-01-05 15:20:04.813 23953466 0 10355664 2014-01-02 21:51:00.617 2014-01-02 21:51:00.617 15630807 15663564 69 2006-10-24 15:12:15.247 2011-01-05 15:23:37.350 80 15706101 10 2011-01-05 15:20:04.810 2011-01-05 15:20:04.810 8355995 8388756 5 2009-01-05 12:16:17.787 2009-01-09 12:54:40.853 8355863 23953467 0 10355664 2014-01-03 08:08:05.860 2014-01-03 08:08:05.860 15706100 10 2011-01-05 15:20:04.810 2011-01-05 15:20:04.810 23953468 0 10355664 2014-01-04 00:55:57.420 2014-01-04 00:55:57.420 15630809 15663566 70 2006-10-24 15:12:15.247 2011-01-05 15:27:54.043 80 15706103 10 2011-01-05 15:20:04.813 2011-01-05 15:20:04.813 8355997 8388758 6 2009-01-05 12:16:17.787 2009-01-09 13:41:21.740 8355863 23953469 0 10355664 2014-01-09 19:09:46.223 2014-01-09 19:09:46.223 15706102 10 2011-01-05 15:20:04.810 2011-01-05 15:20:04.810 8356008 8388769 13 2009-01-05 12:16:17.787 2009-01-09 16:20:29.863 8355863 23953470 0 10355664 2014-01-12 06:17:04.203 2014-01-12 06:17:04.203 8356007 8388768 12 2009-01-05 12:16:17.787 2009-01-09 16:20:11.603 8355863 15630796 15663553 63 2006-10-24 15:12:15.247 2011-01-05 13:10:03.943 80 8356010 8388771 14 2009-01-05 12:16:17.787 2009-01-09 16:21:30.987 8355863 15630798 15663555 64 2006-10-24 15:12:15.247 2011-01-05 15:15:29.713 80 13434985 91 2010-06-09 08:49:22.773 2010-06-09 08:49:22.773 8356004 8388765 10 2009-01-05 12:16:17.787 2009-01-09 16:12:30.300 8355863 8356003 8388764 9 2009-01-05 12:16:17.787 2009-01-09 16:04:08.480 8355863 15706108 10 2011-01-05 15:20:04.813 2011-01-05 15:20:04.813 15630800 15663557 46 2006-10-17 21:39:33.643 2008-10-21 16:16:46.277 10 13434987 91 2010-06-09 08:49:22.777 2010-06-09 08:49:22.777 8356006 8388767 11 2009-01-05 12:16:17.787 2009-01-09 16:14:41.333 8355863 15630801 15663558 65 2006-10-24 15:12:15.247 2011-01-05 15:16:48.337 80 13434986 91 2010-06-09 08:49:22.773 2010-06-09 08:49:22.773 8355983 8388744 1 2009-01-08 16:16:54.793 2009-01-08 16:16:54.793 8355981 25362798 0 10355664 2014-03-03 03:33:45.370 2014-03-03 03:33:45.370 8355985 8388746 2 2009-01-08 16:16:54.793 2009-01-09 09:11:55.607 8355981 8355986 8388747 3 2009-01-08 16:16:54.793 2009-01-09 09:14:07.600 8355981 671 674 15630812 15663569 72 2006-10-24 15:12:15.247 2011-01-05 15:32:51.220 80 8355992 8388753 3 2009-01-05 12:16:17.787 2009-01-06 13:07:14.153 8355863 15630811 15663568 71 2006-10-24 15:12:15.247 2011-01-05 15:29:37.743 80 8355993 8388754 4 2009-01-05 12:16:17.787 2009-01-09 12:54:12.483 8355863 8912925 8945692 11 2009-01-08 16:16:54.793 2009-01-21 14:13:38.747 8355981 8355988 8388749 4 2009-01-08 16:16:54.793 2009-01-09 09:14:30.447 8355981 23953492 0 10355664 2014-01-12 23:31:39.003 2014-01-12 23:31:39.003 8355990 8388751 5 2009-01-08 16:16:54.793 2009-01-09 09:33:14.497 8355981 23953458 0 10355664 2013-12-27 07:35:04.760 2013-12-27 07:35:04.760 23953459 0 10355664 2013-12-28 07:20:28.460 2013-12-28 07:20:28.460 23953460 0 10355664 2013-12-29 07:21:37.387 2013-12-29 07:21:37.387 23953454 0 10355664 2013-12-16 12:38:03.333 2013-12-16 12:38:03.333 8356152 8388905 40 2006-10-24 15:12:15.247 2009-01-13 13:13:57.873 80 23953455 0 10355664 2013-12-17 21:26:16.023 2013-12-17 21:26:16.023 23953456 0 10355664 2013-12-18 09:04:49.663 2013-12-18 09:04:49.663 23953450 0 10355664 2013-12-07 18:03:16.467 2013-12-07 18:03:16.467 23953451 0 10355664 2013-12-10 16:02:49.397 2013-12-10 16:02:49.397 23953452 0 10355664 2013-12-15 05:19:56.833 2013-12-15 05:19:56.833 23953446 0 10355664 2013-12-02 08:48:02.347 2013-12-02 08:48:02.347 23953447 0 10355664 2013-12-02 20:45:29.510 2013-12-02 20:45:29.510 23953448 0 10355664 2013-12-04 04:12:41.267 2013-12-04 04:12:41.267 23953449 0 10355664 2013-12-05 15:15:53.973 2013-12-05 15:15:53.973 8356163 8388916 20 2008-12-18 11:24:49.237 2009-01-13 15:23:06.107 8061037 23953442 0 10355664 2013-11-30 16:44:17.333 2013-11-30 16:44:17.333 25034792 0 10355664 2014-02-12 21:03:01.877 2014-02-12 21:03:01.877 23953440 0 10355664 2013-11-29 16:42:43.057 2013-11-29 16:42:43.057 23953437 0 10355664 2013-11-28 22:43:36.087 2013-12-18 18:20:42.037 23953436 0 10355664 2013-11-28 15:04:52.013 2013-11-28 15:04:52.013 25034802 0 10355664 2014-02-12 21:38:45.110 2014-02-12 21:38:45.110 8356160 8388913 18 2008-12-18 11:24:49.237 2008-12-18 16:27:18.643 8061037 23953433 0 10355664 2013-11-27 01:16:39.773 2013-11-27 01:16:39.773 8356161 8388914 19 2008-12-18 11:24:49.237 2009-01-13 15:20:11.767 8061037 23953432 0 10355664 2013-11-26 20:42:53.163 2013-11-26 20:42:53.163 48 2 2006-10-17 21:39:33.637 2006-10-17 21:39:33.637 23953428 0 10355664 2013-11-26 03:50:50.777 2013-11-26 03:50:50.777 47 2 2006-10-17 21:39:33.637 2006-10-17 21:39:33.637 23953429 0 10355664 2013-11-26 10:35:35.057 2013-11-26 10:35:35.057 46 2 2006-10-17 21:39:33.637 2006-10-17 21:39:33.637 6750218 6782986 5 2006-10-24 21:49:36.650 2008-06-17 16:48:30.587 91 45 2 2006-10-17 21:39:33.627 2006-10-17 21:39:33.627 6750211 6782979 58 2007-06-22 13:07:08.573 2008-06-03 13:17:21.843 1179738 52 2 2006-10-17 21:39:33.637 2006-10-17 21:39:33.637 51 2 2006-10-17 21:39:33.637 2006-10-17 21:39:33.637 23953425 0 10355664 2013-11-25 12:29:08.323 2013-12-29 07:45:32.147 6750213 6782981 59 2007-06-22 13:07:08.573 2008-10-21 16:13:50.257 1179738 50 2 2006-10-17 21:39:33.637 2006-10-17 21:39:33.637 23953422 0 10355664 2013-11-22 12:36:53.277 2013-11-22 12:36:53.277 49 2 2006-10-17 21:39:33.637 2006-10-17 21:39:33.637 23953423 0 10355664 2013-11-22 13:15:23.743 2013-11-22 13:15:23.743 6750214 6782982 45 2006-10-17 21:39:33.643 2008-08-13 09:06:55.550 10 23953420 0 10355664 2013-11-22 01:36:23.183 2013-11-22 01:36:23.183 23953418 0 10355664 2013-11-21 03:05:10.463 2013-11-21 03:05:10.463 23953416 0 10355664 2013-11-20 03:46:51.913 2013-11-20 03:46:51.913 17891620 17858877 23953415 0 10355664 2013-11-20 03:23:50.667 2013-11-20 03:23:50.667 23953412 0 10355664 2013-11-19 13:44:57.863 2013-11-19 13:44:57.863 9798394 9831144 15 2006-10-24 21:49:36.650 2008-12-30 13:29:53.517 91 838 login result page /loginResult.jsp Engineering Distribution List Research Distribution List ITD Distribution List DMO Distribution List OED Distribution List {noformat} # Whew, once all that is done, you are ready to build and deploy the SSDS. Open command prompt, cd to directory where SSDS was checked out and type {noformat} ant -Dtarget=deploy {noformat} # Once the build is complete, open a terminal window, change directory to where the JBoss home is, and start JBoss {noformat} sudo ./bin/run.sh {noformat} # Once JBoss finishes startup, you should see the line: {noformat} 17:12:41,607 INFO [Server] JBoss (MX MicroKernel) [4.2.2.GA (build: SVNTag=JBoss_4_2_2_GA date=200710221139)] Started in 19s:45ms {noformat} and hopefully you did not see any stack traces fly by during startup. If all looks OK, you should be able to go the SSDS_Metadata database and see newly created tables that will be where SSDS will store is metadata. You can also open a browser and go to [http://localhost:8080/ssds-docs/] to see a simple page with some links to SSDS documentation and various utilities and libraries. {note:title=Still to document} # Configure mod_jk in Apache/JBoss # Configure SSL for login.jsp page {note} h3. Setup an Eclipse project: In order to actually do some development on the SSDS code base, you can use an IDE like Eclipse to edit the code and use ant to deploy your changes to JBoss. To setup an Eclipse project. # Install eclipse that you download from [http://www.eclipse.org] # Startup Eclipse and select File->New->Java Project. # In the New Project Window, select 'Create project from existing source' and browse to where you checked out the SSDS source code. Once you configure the project correctly, you should have the following project settings: ## There should be two source directories: {noformat} src/java {noformat} and {noformat} src/gen {noformat} (src/gen is created by ant during build) # Add all jars in the lib directory # Add src/resources/build/antlr/antlr-2.7.5.jar # Add JBoss jars ## JBOSS_HOME/client/activation.jar ## JBOSS_HOME/client/servlet-api.jar ## JBOSS_HOME/client/jboss-j2ee.jar ## JBOSS_HOME/client/log4j.jar ## JBOSS_HOME/server/default/lib/commons-codec.jar ## JBOSS_HOME/server/default/lib/commons-collections.jar ## JBOSS_HOME/server/default/lib/commons-httpclient.jar ## JBOSS_HOME/server/default/lib/hibernate3.jar ## JBOSS_HOME/server/default/lib/mail.jar # output set to build/classes (to align with ant's build files) h3. Create a FlexBuilder (plug-in) project: # Start Eclipse with Flex-Builder plug-in installed # File->New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/build/flex and choose it for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). {note:title=Will create a Main.mxml} Note that the creation of the Flex Builder project will create a Main.mxml file. To fix this, right click on explorer.mxml and choose 'Set As Default Application', then you can delete the Main.mxml file. {note} # Click on Finish NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 841 8356088 8388845 34 2009-01-05 12:16:17.787 2009-01-12 12:52:08.050 8355863 8356087 8388844 38 2006-10-24 15:12:15.247 2009-01-10 19:52:24.737 80 10355406 10388137 23 2007-02-08 12:50:50.907 2009-07-16 12:59:17.527 841 8356084 8388841 32 2009-01-05 12:16:17.787 2009-01-12 12:42:48.067 8355863 8356086 8388843 33 2009-01-05 12:16:17.787 2009-01-12 12:44:46.293 8355863 8356080 8388837 30 2009-01-05 12:16:17.787 2009-01-12 12:21:15.023 8355863 10355411 10388142 24 2007-02-08 12:50:50.907 2009-07-16 15:59:36.620 841 8356082 8388839 31 2009-01-05 12:16:17.787 2009-01-12 12:32:10.280 8355863 10355413 10388144 25 2007-02-08 12:50:50.907 2009-07-16 16:30:25.937 841 8356076 8388833 28 2009-01-05 12:16:17.787 2009-01-12 11:40:45.710 8355863 10355415 10388146 26 2007-02-08 12:50:50.907 2009-07-16 16:39:07.153 841 10355416 10388147 27 2007-02-08 12:50:50.907 2009-07-16 16:40:02.687 841 8356078 8388835 29 2009-01-05 12:16:17.787 2009-01-12 12:12:44.283 8355863 10355417 10388148 28 2007-02-08 12:50:50.907 2009-07-16 16:40:28.880 841 10355419 10388150 29 2007-02-08 12:50:50.907 2009-07-16 16:40:51.567 841 10355421 10388152 30 2007-02-08 12:50:50.907 2009-07-16 16:48:04.143 841 10389125 10356413 10355423 10388154 31 2007-02-08 12:50:50.907 2009-07-16 16:50:55.653 841 10355426 10388157 32 2007-02-08 12:50:50.907 2009-07-16 16:54:46.260 841 10355429 10388160 33 2007-02-08 12:50:50.907 2009-07-16 16:57:21.967 841 10355431 10388162 34 2007-02-08 12:50:50.907 2009-07-16 17:02:15.930 841 10355433 10388164 35 2007-02-08 12:50:50.907 2009-07-16 17:12:06.060 841 8945734 DataProducer that must be of type "deployment" and then creates "deep" copy of it and persists the new copy. It pulls in the options specified in the incoming parameters to create a unique copy of the DataProducers and DataContainer (outputs). All the rest of the object should be linked to the ones that already exist in the peristent storage. The new ID of the duplicate deployment is returned. h5. The Interface ||Return||Parameters|| |The ID of the newly generated deployment|* DataProducer to Copy * Date the copy should start at * A boolean to indicate if the old (original) should be closed * Date the old (original) should use as an end date (if to be closed) * String that will be the name of the new deployment (copy) * String that is the base of the data stream URI which should be {code}http://new-ssds.mbari.org/servlet/GetOriginalDataServlet{code}| So the API interface looks something like: {code:title=createDuplicateDeepDeployment} public Long createDuplicateDeepDeployment( DataProducer deploymentToCopy, // DataProducer to copy Date newStartDate, // Date to start copied DataProducer on boolean closeOld, // Whether or not old (original) should be closed Date oldEndDate, // Date to use for end date if original is closed String newHeadDeploymentName, // Name of the new DataProducer (copy) String baseDataStreamUri // Base URI for raw data access (http://new-ssds.mbari.org/servlet/GetOriginalDataServlet) ) {code} h5. Java EJB client Under construction h5. REST Style client To use this service via HTTP (REST Style), the call would be constructed using this format: {code:title=REST Style format} http://new-ssds.mbari.org/servlet/MetadataAccessServlet?objectToInvokeOn=DataProducerAccess&method=createDuplicateDeepDeployment &p1Type=DataProducer&p1Value=DataProducer|id=1938292 &p2Type=Date&p2Value=2008-12-01T00:00:00Z &p3Type=boolean&p3Value=true &p4Type=Date&p4Value=2008-11-30T00:00:00Z &p5Type=String&p5Value=New copy of Deployment &p6Type=String&p6Value=http://new-ssds.mbari.org/servlet/GetOriginalDataServlet {code} And the return might looks something like this: {code} Result(s): java.lang.Long|29985 {code} h5. Web Services client Under construction {panel} {anchor:makePersistent} {panel:title=makePersistent} h5. Background The purpose of this service is to insert (or update) a DataProducer in the SSDS. If the DataProducer already exists (the equal criteria is by ID and then if no ID specified, it uses the outputs) it will update it, if not it will create a new one. h5. The Interface ||Return||Parameters|| |The ID of the created or updated DataProducer|* DataProducer to persist| So the API interface looks something like: {code:title=createDuplicateDeepDeployment} public Long makePersistent( DataProducer dataProducerToPersist // This is the DataProducer that will be updated or created ) {code} h5. Java EJB client Under construction h5. REST Style client To use this service via HTTP (REST Style), the call would be constructed using this format: {code:title=REST Style format} http://new-ssds.mbari.org/servlet/MetadataAccessServlet?objectToInvokeOn=DataProducerAccess&method=makePersistent &p1Type=DataProducer&p1Value=DataProducer|name=MUCE - January 2009 test|description=MUCE Test Deployment|dataProducerType=Deployment|startDate=2009-01-10T12:00:00Z {code} And the return might looks something like this: {code} Result(s): java.lang.Long|29991 {code} h5. Web Services client Under construction {panel} ]]> 8912969 8945735 0, the service will consider any time between // samples that is less than the specified gap plus this margin, // it will assume that it is not a gap condition. This is to // prevent false positives when the sample timestamps aren't exactly // on the interval. String gapSpec, // There are two ways to specify a gap: // "serviceCalculated" or "userSpecified" Long numberOfRecords, // If the call specifies SERVICE_CALCULATED and this // is greater than 0, the service will use 'numberOfRecords' // most recent records of the specified RecordType // in calculating the average time between samples Date intervalCalcStartWindow, // This is the date that starts the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this is ignored. Date intervalCalcEndWindow, // This is the date that ends the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this will be used as // the endtime and then the service will use the numberOfRecords // before this time as the data to calculate the average // time interval. Long gapInMillis // If the gapSpec is USER_SPECIFIED, then the service will use // this number of milliseconds as the gap for identifying gaps. ) {code} With a return that has the format of: Properties Objects with properties: ||Property Name||Value|| |lastPacketDateTime|This is the date and time of the last packet received| |totalNumberOfRecords|This is the total number of records for the parameters specified| |numberOfFuturePackets|This is the number of packet that appear in the future. This should be zero and if they are not, there could be bad data| |averagSampleIntervalInMillis|This is the number of milliseconds that the service used to find data gaps| |marginInMillis|This is the number of milliseconds as a margin that the service used to find data gaps| |numRecordsSearchedForGaps|This is the number of records that were searched through while trying to find gaps using the gap criteria| |dataGap1Start|This is the date and time of start of the first possible gap in the data| |dataGap1End|This is the date and time of end of the first possible gap in the data| |.|.| |.|.| |.|.| |dataGapNStart|This is the date and time of start of the Nth possible gap in the data| |dataGapNEnd|This is the date and time of end of the Nth possible gap in the data| h5. Java EJB client Under construction h5. REST client If you want to use the REST-style interface the HTTP call would looks something like: {code} http://new-ssds.mbari.org:8080/servlet/DataAccessServlet?objectToInvokeOn=SQLDataStreamRawDataAccess&method=getDataStreamProperties &p1Type=Long&p1Value=1300 &p2Type=Long&p2Value=1 &p3Type=Boolean&p3Value=true &p4Type=Date&p4Value=2008-12-00T00:00:00Z &p5Type=Date&p5Value=2008-01-00T00:00:00Z &p6Type=String&p6Value=timeGap &p7Type=Long&p7Value=5000 &p8Type=String&p8Value=userSpecified &p9Type=Long&p9Value=10 &p10Type=Date&p10Value= &p11Type=Date&p11Value= &p12Type=Long&p12Value=10000 {code} And the return might look something like {code} java.util.Properties{} {code} h5. Web Service Client Under construction {panel}]]> 8912971 8356116 8388873 7 2009-01-08 16:16:54.793 2009-01-13 10:12:26.593 8355981 11829811 11797058 11829826 ALOHA interactions. h5. Timeline: More details to come from ALOHA group, but some timeline notes: # Ship time is January # They want test system in lab ASAP (they actually have some of it running currently) h5. Action Items: # Kevin will look back and try to get SSDS install going again on kainani. # ALOHA group will send Bob information about instruments (IP addresses, etc.) # Bob will look into getting SIAM running against their instruments. # ALOHA group will work on generating a timeline with milestones like code freeze, integration and test, etc. NOTE: As long as the time is not huge, we can charge to TTOS.]]> 11797073 8356114 8388871 6 2009-01-08 16:16:54.793 2009-01-09 10:41:42.270 8355981 11829824 11797071 19300614 2 2012-06-18 17:31:59.780 2012-06-18 17:31:59.780 8356136 8388892 39 2006-10-24 15:12:15.247 2009-01-12 12:52:59.973 80 10355400 10388131 22 2007-02-08 12:50:50.907 2009-01-27 09:19:45.273 841 10458414 10355890 2009-08-11 16:52:13.333 2009-08-11 16:52:13.333 9372424 9405156 8 2007-08-03 07:17:05.643 2008-11-20 13:15:29.247 1179945 16777259 16810027 7 2010-04-27 15:25:51.250 2011-04-14 14:35:33.237 11797071 13205733 11240414 2010-06-03 17:11:55.370 2010-06-03 17:11:55.370 13205732 11240414 2010-06-03 17:11:55.370 2010-06-03 17:11:55.370 13205734 11240414 2010-06-03 17:11:55.370 2010-06-03 17:11:55.370 13205730 11240414 2010-06-03 17:11:55.370 2010-06-03 17:11:55.370 13205731 11240414 2010-06-03 17:11:55.370 2010-06-03 17:11:55.370 8356216 8388964 8 2009-01-08 16:16:54.793 2009-01-13 10:13:53.603 8355981 8356217 8388965 9 2009-01-08 16:16:54.793 2009-01-15 12:51:47.440 8355981 58 2 2006-10-17 21:39:33.637 2006-10-17 21:39:33.637 57 2 2006-10-17 21:39:33.637 2006-10-17 21:39:33.637 25363009 0 10355664 2014-03-05 14:25:33.280 2014-03-05 14:25:33.280 54 2 2006-10-17 21:39:33.637 2006-10-17 21:39:33.637 53 2 2006-10-17 21:39:33.637 2006-10-17 21:39:33.637 56 2 2006-10-17 21:39:33.637 2006-10-17 21:39:33.637 21495819 0 10355664 2013-08-02 06:57:34.263 2013-08-25 02:38:23.933 55 2 2006-10-17 21:39:33.637 2006-10-17 21:39:33.637 21495828 0 10355664 2013-08-22 08:05:09.607 2013-12-05 22:38:53.747 21495826 0 10355664 2013-08-15 20:20:13.553 2013-08-15 20:20:13.553 2162927 2195693 3 2007-08-03 07:17:05.643 2007-08-03 09:44:52.413 1179945 21495827 0 10355664 2013-08-17 05:20:02.903 2013-08-17 05:20:02.903 22446083 0 10355664 2013-09-08 11:15:53.200 2014-01-01 00:18:08.023 22446082 0 10355664 2013-09-08 02:34:08.200 2013-09-08 02:34:08.200 14024905 13664436 2010-07-28 08:02:19.663 2010-07-28 08:02:19.663 16777255 16810023 5 2010-04-27 15:25:51.250 2011-04-14 14:31:58.783 11797071 16777257 16810025 6 2010-04-27 15:25:51.250 2011-04-14 14:34:45.330 11797071 17170436 17236005 2011-06-02 09:36:34.580 2011-06-02 09:36:34.580 51203 1 16777251 16810019 3 2010-04-27 15:25:51.250 2011-04-14 14:16:53.337 11797071 17170437 17236005 2011-06-02 09:36:43.503 2011-06-02 09:36:43.503 51203 1 16777253 16810021 4 2010-04-27 15:25:51.250 2011-04-14 14:17:09.927 11797071 17170435 17236005 2011-06-02 00:01:14.877 2011-06-02 00:01:14.877 26627 1 17170440 17236005 2011-06-02 11:50:08.487 2011-06-02 11:50:08.487 29144 1 17170438 17170441 17236005 2011-06-02 12:04:19.047 2011-06-02 12:04:19.047 10455 1 16777248 16810016 1 2010-04-27 15:25:51.250 2010-04-27 15:25:51.250 11797071 17170438 17236005 2011-06-02 11:50:08.487 2011-06-02 11:53:48.827 18725 2 17170439 17236005 2011-06-02 11:50:17.740 2011-06-02 11:50:17.740 31945 1 16777250 16810018 2 2010-04-27 15:25:51.250 2011-04-14 13:46:17.857 11797071 17170442 17236005 2011-06-02 12:09:15.227 2011-06-02 12:09:15.227 12486 1 11797286 11830030 63 2009-01-05 12:16:17.787 2010-05-05 18:10:57.723 8355863 3604482 1 80 2008-04-11 08:28:15.050 2008-04-11 08:28:15.050 11797284 11830028 62 2009-01-05 12:16:17.787 2010-05-05 17:55:48.197 8355863 11797289 11830033 65 2009-01-05 12:16:17.787 2010-05-05 22:11:43.553 8355863 11797287 11830031 64 2009-01-05 12:16:17.787 2010-05-05 18:13:56.557 8355863 11797277 11830021 58 2009-01-05 12:16:17.787 2010-05-05 17:05:51.883 8355863 11797275 11830019 57 2009-01-05 12:16:17.787 2010-05-05 17:02:14.650 8355863 11797282 11830026 61 2009-01-05 12:16:17.787 2010-05-05 17:30:15.387 8355863 11797280 11830024 60 2009-01-05 12:16:17.787 2010-05-05 17:29:38.237 8355863 11797279 11830023 59 2009-01-05 12:16:17.787 2010-05-05 17:16:14.583 8355863 11797269 11830013 54 2009-01-05 12:16:17.787 2010-05-05 16:57:14.473 8355863 11797268 11830012 53 2009-01-05 12:16:17.787 2010-05-05 16:55:46.863 8355863 11797273 11830017 56 2009-01-05 12:16:17.787 2010-05-05 16:59:12.157 8355863 11797271 11830015 55 2009-01-05 12:16:17.787 2010-05-05 16:57:54.247 8355863 11797261 11830005 49 2009-01-05 12:16:17.787 2010-05-05 16:32:42.917 8355863 6848528 1179738 2008-10-21 16:16:10.390 2008-10-21 16:16:10.390 11797262 11830006 50 2009-01-05 12:16:17.787 2010-05-05 16:52:48.507 8355863 6848529 1179738 2008-10-21 16:16:10.390 2008-10-21 16:16:10.390 6848530 1179738 2008-10-21 16:16:10.390 2008-10-21 16:16:10.390 6848531 1179738 2008-10-21 16:16:10.393 2008-10-21 16:16:10.393 6848524 1179738 2008-10-21 16:16:10.390 2008-10-21 16:16:10.390 11797266 11830010 52 2009-01-05 12:16:17.787 2010-05-05 16:54:15.453 8355863 6848525 1179738 2008-10-21 16:16:10.390 2008-10-21 16:16:10.390 6848526 1179738 2008-10-21 16:16:10.390 2008-10-21 16:16:10.390 11797264 11830008 51 2009-01-05 12:16:17.787 2010-05-05 16:53:26.140 8355863 6848527 1179738 2008-10-21 16:16:10.390 2008-10-21 16:16:10.390 11042902 8355863 2009-01-05 12:16:39.663 2010-01-05 12:19:46.223 17405 11 8519681 11797256 11830000 47 2009-01-05 12:16:17.787 2010-05-05 15:55:43.797 8355863 11797255 11829999 46 2009-01-05 12:16:17.787 2010-04-29 11:59:15.357 8355863 11797258 11830002 48 2009-01-05 12:16:17.787 2010-05-05 15:56:12.390 8355863 6848533 1179738 2008-10-21 16:16:10.393 2008-10-21 16:16:10.393 6848532 1179738 2008-10-21 16:16:10.393 2008-10-21 16:16:10.393 11042896 8355863 2009-01-05 12:16:39.663 2010-01-04 16:40:08.197 15630 6 8519681 11042897 8355863 2009-01-05 12:16:39.663 2010-01-04 16:56:48.793 16838 7 8519681 11042895 8355863 2009-01-05 12:16:39.663 2009-01-05 17:26:19.823 14015 5 8519681 11042901 8355863 2009-01-05 12:16:39.663 2010-01-04 17:01:00.750 17417 10 8519681 11042898 8355863 2009-01-05 12:16:39.663 2010-01-04 16:58:29.700 16551 8 8519681 11042899 8355863 2009-01-05 12:16:39.663 2010-01-04 17:00:28.953 17429 9 8519681 11042893 8355863 2010-01-04 14:20:46.573 2010-01-04 14:20:46.573 112595 1 11042891 8355863 2009-01-09 14:25:41.067 2009-01-09 22:23:39.220 126700 2 8519698 10945007 10912252 11042862 10911848 2009-10-29 09:41:41.037 2009-10-29 10:23:29.117 99838 4 11042854 11042863 10911848 2009-10-29 09:41:41.017 2009-10-29 10:28:51.077 2443 5 11042853 11042864 10911848 2009-10-29 09:41:41.037 2009-10-29 10:28:51.093 113423 5 11042854 11042859 10911848 2009-10-29 09:41:41.017 2009-10-29 10:06:05.023 2355 3 11042853 11042858 10911848 2009-10-29 09:41:41.037 2009-10-29 09:44:49.950 77389 2 11042854 11042861 10911848 2009-10-29 09:41:41.017 2009-10-29 10:23:29.107 2305 4 11042853 11042860 10911848 2009-10-29 09:41:41.037 2009-10-29 10:06:05.060 90560 3 11042854 11042855 10911848 2009-10-29 09:41:41.017 2009-10-29 09:41:41.017 1659 1 11042853 11042854 10911848 2009-10-29 09:41:41.037 2009-11-02 09:20:42.167 111714 6 11042857 10911848 2009-10-29 09:41:41.017 2009-10-29 09:44:49.907 1721 2 11042853 11042856 10911848 2009-10-29 09:41:41.037 2009-10-29 09:41:41.037 70696 1 11042854 11042851 91 2009-10-27 10:16:33.607 2009-10-28 16:35:53.747 18755 31 11042820 11042850 91 2009-10-27 10:16:33.607 2009-10-28 16:33:30.837 18955 30 11042820 11042853 10911848 2009-10-29 09:41:41.017 2009-11-02 09:20:42.107 2165 6 11042852 91 2009-10-27 10:16:33.607 2009-10-28 16:39:16.947 18782 32 11042820 11042847 91 2009-10-27 10:16:33.607 2009-10-28 16:28:07.813 19792 27 11042820 11042846 91 2009-10-27 10:16:33.607 2009-10-28 16:26:56.810 19820 26 11042820 8519712 8061037 2009-01-13 15:20:58.300 2009-01-13 15:22:33.870 1439 1 11042849 91 2009-10-27 10:16:33.607 2009-10-28 16:33:01.203 18953 29 11042820 8519710 8355863 2009-01-12 11:13:21.350 2009-01-12 12:27:20.950 21339 10 8519700 11042848 91 2009-10-27 10:16:33.607 2009-10-28 16:29:28.837 19789 28 11042820 8519706 8355863 2009-01-12 11:13:21.350 2009-01-12 11:59:16.940 18935 6 8519700 11042844 91 2009-10-27 10:16:33.607 2009-10-28 16:22:38.383 19757 24 11042820 8519707 8355863 2009-01-12 11:13:21.350 2009-01-12 11:59:41.580 19288 7 8519700 11042845 91 2009-10-27 10:16:33.607 2009-10-28 16:25:38.647 19804 25 11042820 8519708 8355863 2009-01-12 11:13:21.350 2009-01-12 12:02:03.677 19373 8 8519700 11042842 91 2009-10-27 10:16:33.607 2009-10-28 16:20:18.107 19767 22 11042820 8519709 8355863 2009-01-12 11:13:21.350 2009-01-12 12:02:53.957 20909 9 8519700 11042843 91 2009-10-27 10:16:33.607 2009-10-28 16:20:51.663 19762 23 11042820 8519702 8355863 2009-01-12 11:13:21.350 2009-01-12 11:20:16.767 961 2 8519700 11042840 91 2009-10-27 10:16:33.607 2009-10-27 13:56:15.640 19949 20 11042820 8519703 8355863 2009-01-12 11:13:21.350 2009-01-12 11:44:31.323 485 3 8519700 11042841 91 2009-10-27 10:16:33.607 2009-10-28 16:19:18.047 19763 21 11042820 8519704 8355863 2009-01-12 11:13:21.350 2009-01-12 11:50:08.717 15368 4 8519700 25395369 25428137 13 2009-07-31 11:41:34.023 2014-03-05 15:53:41.113 10355664 11042838 91 2009-10-27 10:16:33.607 2009-10-27 13:48:56.510 19190 18 11042820 8519705 8355863 2009-01-12 11:13:21.350 2009-01-12 11:52:50.057 15624 5 8519700 11042839 91 2009-10-27 10:16:33.607 2009-10-27 13:54:46.057 19450 19 11042820 8519698 8355863 2009-01-09 14:25:41.067 2010-01-04 10:06:00.357 195690 3 11042836 91 2009-10-27 10:16:33.607 2009-10-27 13:31:32.977 18876 16 11042820 8519699 8355863 2009-01-09 14:25:41.067 2009-01-09 14:25:41.067 101047 1 8519698 11042837 91 2009-10-27 10:16:33.607 2009-10-27 13:47:28.413 18878 17 11042820 8519700 8355863 2009-01-12 11:13:21.350 2010-04-29 11:49:38.193 21664 12 11042834 91 2009-10-27 10:16:33.607 2009-10-27 13:04:21.210 13775 14 11042820 8519701 8355863 2009-01-12 11:13:21.350 2009-01-12 11:13:21.350 133 1 8519700 11042835 91 2009-10-27 10:16:33.607 2009-10-27 13:29:53.660 16137 15 11042820 8519694 8355863 2009-01-09 12:54:40.847 2009-01-09 13:31:03.813 7919 3 8519691 11042832 91 2009-10-27 10:16:33.607 2009-10-27 12:23:25.007 13792 12 11042820 8519695 8355863 2009-01-09 12:54:40.847 2009-01-09 13:33:15.910 7942 4 8519691 11042833 91 2009-10-27 10:16:33.607 2009-10-27 13:03:17.103 13773 13 11042820 8519696 8355863 2009-01-09 12:54:40.847 2009-01-09 13:36:10.163 8209 5 8519691 11042830 91 2009-10-27 10:16:33.607 2009-10-27 11:44:17.740 13005 10 11042820 8519697 8355863 2009-01-09 12:54:40.847 2009-01-09 13:36:38.960 8168 6 8519691 11042831 91 2009-10-27 10:16:33.607 2009-10-27 11:54:16.527 13792 11 11042820 8519691 8355863 2009-01-09 12:54:40.847 2009-01-09 13:40:26.533 8113 7 11042829 91 2009-10-27 10:16:33.607 2009-10-27 11:26:51.333 11305 9 11042820 11042828 91 2009-10-27 10:16:33.607 2009-10-27 11:18:13.293 10683 8 11042820 11042827 91 2009-10-27 10:16:33.607 2009-10-27 11:17:13.250 10032 7 11042820 8519693 8355863 2009-01-09 12:54:40.847 2009-01-09 13:22:10.487 8477 2 8519691 11042826 91 2009-10-27 10:16:33.607 2009-10-27 11:15:04.980 9382 6 11042820 8519692 8355863 2009-01-09 12:54:40.847 2009-01-09 12:54:40.847 133 1 8519691 11042825 91 2009-10-27 10:16:33.607 2009-10-27 10:45:51.407 8134 5 11042820 11042824 91 2009-10-27 10:16:33.607 2009-10-27 10:20:31.320 3604 4 11042820 11042823 91 2009-10-27 10:16:33.607 2009-10-27 10:19:59.067 3604 3 11042820 11042822 91 2009-10-27 10:16:33.607 2009-10-27 10:16:49.600 534 2 11042820 11042821 91 2009-10-27 10:16:33.607 2009-10-27 10:16:33.607 133 1 11042820 8519683 8355863 2009-01-05 12:16:39.663 2009-01-05 12:17:52.990 615 2 8519681 11042820 91 2009-10-27 10:16:33.607 2009-10-28 16:41:41.487 18736 33 8519682 8355863 2009-01-05 12:16:39.663 2009-01-05 12:16:39.663 133 1 8519681 8519685 8355863 2009-01-05 12:16:39.663 2009-01-05 13:56:16.650 11749 4 8519681 8519684 8355863 2009-01-05 12:16:39.663 2009-01-05 13:53:11.407 9132 3 8519681 11797291 11830035 66 2009-01-05 12:16:17.787 2010-05-05 22:40:03.890 8355863 10461536 10356008 2009-09-30 13:48:58.870 2009-09-30 13:48:58.870 11797292 11830036 67 2009-01-05 12:16:17.787 2010-05-05 22:43:41.243 8355863 10461537 10356008 2009-09-30 13:48:58.870 2009-09-30 13:48:58.870 8519681 8355863 2009-01-05 12:16:39.663 2010-06-08 10:31:50.700 20538 19 14647300 1 10355664 2010-08-25 09:45:15.077 2010-08-25 09:45:15.077 5374178 5406926 4 2006-10-24 21:49:36.650 2006-10-24 21:56:46.370 91 17858875 17891618 76 2006-10-24 15:12:15.247 2011-05-30 23:12:56.577 80 17858879 17891622 1 2011-07-15 08:16:34.157 2011-07-15 08:16:34.157 17858877 10322148 1 10355664 2009-09-09 19:56:15.077 2009-09-09 19:56:15.077 10322158 1 10355664 2009-09-14 13:37:15.087 2009-09-14 13:37:15.087 3113031 3178563 38 2006-10-17 21:39:33.643 2007-06-22 12:54:46.797 10 10322168 2 10355664 2009-09-19 08:14:15.050 2009-09-25 15:56:15.047 3638118 3670857 34 2008-05-07 14:59:30.507 2008-05-16 11:32:08.647 3637922 3638116 3670855 33 2008-05-07 14:59:30.507 2008-05-16 11:23:37.357 3637922 3638121 3670860 36 2008-05-07 14:59:30.507 2008-05-16 11:56:36.150 3637922 10356010 10388736 1 2009-08-17 09:07:20.000 2009-08-17 09:07:20.000 10356008 3638119 3670858 35 2008-05-07 14:59:30.507 2008-05-16 11:44:30.567 3637922 10322170 1 10355664 2009-09-21 13:21:15.083 2009-09-21 13:21:15.083 11013138 10912252 2009-12-28 13:54:14.580 2009-12-28 13:54:14.580 3638114 3670853 32 2008-05-07 14:59:30.507 2008-05-12 11:26:43.863 3637922 10461644 8061037 2009-09-30 16:19:18.870 2009-09-30 16:19:18.870 4390913 1 10 2008-05-28 16:21:15.083 2008-05-28 16:21:15.083 10461645 8061037 2009-09-30 16:19:18.870 2009-09-30 16:19:18.870 3638020 3670767 23 2008-05-07 14:59:30.507 2008-05-08 16:30:40.587 3637922 3638022 3670769 24 2008-05-07 14:59:30.507 2008-05-08 16:58:43.290 3637922 10322082 1 10355664 2009-08-05 15:27:15.043 2009-08-05 15:27:15.043 3638016 3670763 21 2008-05-07 14:59:30.507 2008-05-08 16:08:56.170 3637922 3638018 3670765 22 2008-05-07 14:59:30.507 2008-05-08 16:22:36.500 3637922 3638012 3670759 19 2008-05-07 14:59:30.507 2008-05-08 14:56:59.643 3637922 10461643 8061037 2009-09-30 16:19:18.870 2009-09-30 16:19:18.870 1933368 1 1179945 2007-10-04 11:10:15.357 2007-10-04 11:10:15.357 3638014 3670761 20 2008-05-07 14:59:30.507 2008-05-08 15:23:49.447 3637922 1933369 1 10 2007-10-04 11:50:15.047 2007-10-04 11:50:15.047 3638008 3670755 17 2008-05-07 14:59:30.507 2008-05-08 14:46:47.267 3637922 3638007 3670754 16 2008-05-07 14:59:30.507 2008-05-08 14:07:08.347 3637922 3638010 3670757 18 2008-05-07 14:59:30.507 2008-05-08 14:55:08.033 3637922 10322092 1 10355664 2009-08-11 12:00:15.083 2009-08-11 12:00:15.083 3638003 3670750 14 2008-05-07 14:59:30.507 2008-05-08 12:32:36.037 3637922 3638005 3670752 15 2008-05-07 14:59:30.507 2008-05-08 14:03:41.373 3637922 3637995 3670742 13 2008-05-07 14:59:30.507 2008-05-08 12:12:29.620 3637922 3638047 3670794 31 2008-05-07 14:59:30.507 2008-05-12 11:18:11.937 3637922 3638045 3670792 30 2008-05-07 14:59:30.507 2008-05-12 11:06:28.867 3637922 16515520 8355863 2011-03-18 09:12:36.237 2011-03-18 09:12:36.237 16515521 8355863 2011-03-18 09:12:36.237 2011-03-18 09:12:36.237 3638043 3670790 29 2008-05-07 14:59:30.507 2008-05-12 10:54:07.157 3637922 16515522 8355863 2011-03-18 09:12:36.237 2011-03-18 09:12:36.237 16515523 8355863 2011-03-18 09:12:36.237 2011-03-18 09:12:36.237 1933373 1 10 2007-10-04 11:52:15.053 2007-10-04 11:52:15.053 3638041 3670788 28 2008-05-07 14:59:30.507 2008-05-12 10:27:42.917 3637922 3638039 3670786 27 2008-05-07 14:59:30.507 2008-05-12 10:18:24.887 3637922 1933370 2 80 2007-10-04 11:51:15.027 2007-10-04 11:51:15.027 3638037 3670784 26 2008-05-07 14:59:30.507 2008-05-12 09:51:33.283 3637922 1343511 4 596 2007-07-03 09:49:15.067 2007-10-04 11:50:15.030 1343512 2 1179794 2007-07-03 09:49:15.080 2007-07-03 09:56:15.017 1343513 1 674 2007-07-03 09:49:15.080 2007-07-03 09:49:15.080 1343502 1 10 2007-06-08 14:38:15.030 2007-06-08 14:38:15.030 3638029 3670776 25 2008-05-07 14:59:30.507 2008-05-08 17:02:43.757 3637922 10912281 10945036 1 2010-01-04 14:31:35.077 2010-01-04 14:31:35.077 10912280 3637934 3670683 3 2008-05-07 14:59:30.507 2008-05-07 15:10:44.727 3637922 10912282 10945037 2 2010-01-04 14:31:35.077 2010-01-04 14:31:52.047 10912280 10912277 10945032 37 2009-01-05 12:16:17.787 2010-01-04 09:32:52.437 8355863 3637938 3670687 5 2008-05-07 14:59:30.507 2008-05-07 15:29:13.427 3637922 10912278 10945033 38 2009-01-05 12:16:17.787 2010-01-04 13:44:38.453 8355863 3637936 3670685 4 2008-05-07 14:59:30.507 2008-05-07 15:28:13.110 3637922 3637942 3670691 7 2008-05-07 14:59:30.507 2008-05-07 16:09:56.163 3637922 3637940 3670689 6 2008-05-07 14:59:30.507 2008-05-07 15:39:20.390 3637922 3637946 3670695 9 2008-05-07 14:59:30.507 2008-05-07 17:09:48.657 3637922 3637944 3670693 8 2008-05-07 14:59:30.507 2008-05-07 16:36:27.030 3637922 3637949 3670698 10 2008-05-07 14:59:30.507 2008-05-08 10:06:51.233 3637922 10912295 10945050 46 2006-10-24 15:12:15.247 2010-01-04 16:17:35.743 80 10912293 10945048 2 2009-01-12 13:08:50.467 2010-01-04 16:17:35.820 8356090 10912291 10945046 1 2009-01-12 13:08:50.467 2009-01-12 13:08:50.467 8356090 10912290 10945045 5 2010-01-04 14:31:35.077 2010-01-04 15:10:23.837 10912280 10912289 10945044 45 2006-10-24 15:12:15.247 2010-01-04 14:36:09.820 80 10912288 10945043 4 2010-01-04 14:31:35.077 2010-01-04 14:36:35.997 10912280 10912286 10945041 3 2010-01-04 14:31:35.077 2010-01-04 14:33:00.657 10912280 10912285 10945040 44 2006-10-24 15:12:15.247 2010-01-04 14:35:37.247 80 10912284 10945039 43 2006-10-24 15:12:15.247 2009-08-13 11:26:51.553 80 22806529 0 10355664 2013-09-27 12:59:40.927 2013-09-27 12:59:40.927 10912264 10945019 36 2009-01-05 12:16:17.787 2009-10-07 10:40:30.950 8355863 10485950 10355890 2009-08-11 16:42:24.317 2009-08-11 16:42:24.317 514578 1 11013509 10912280 2010-01-04 16:17:35.810 2010-01-04 16:17:35.810 3637988 3670735 11 2008-05-07 14:59:30.507 2008-05-08 11:18:20.140 3637922 3637992 3670739 12 2008-05-07 14:59:30.507 2008-05-08 12:10:11.680 3637922 10322228 1 10355664 2009-10-08 21:22:15.097 2009-10-08 21:22:15.097 11797075 11829828 1 2010-04-27 15:32:49.873 2010-04-27 15:32:49.873 11797073 11797077 11829830 2 2010-04-27 15:32:49.873 2010-04-27 15:33:29.950 11797073 11797078 11829831 49 2006-10-24 15:12:15.247 2010-04-27 15:24:47.257 80 10355897 10388623 4 2009-08-11 16:05:27.267 2009-08-11 16:38:15.483 10355890 10355898 10388624 5 2009-08-11 16:05:27.267 2009-08-11 16:39:55.963 10355890 10355895 10388621 3 2009-08-11 16:05:27.267 2009-08-11 16:16:51.717 10355890 10485949 10355890 2009-08-11 16:37:51.760 2009-08-11 16:37:51.760 217375 1 10355894 10388620 2 2009-08-11 16:05:27.267 2009-08-11 16:06:45.350 10355890 10355891 10388617 1 2009-08-11 16:05:27.267 2009-08-11 16:06:45.337 10355890 11797069 11829822 48 2006-10-24 15:12:15.247 2010-04-26 14:58:36.523 80 10912357 10945103 6 2009-01-22 10:36:03.900 2010-01-07 17:00:06.230 8912971 10355902 10388628 7 2009-08-11 16:05:27.267 2009-08-11 16:46:37.537 10355890 10355900 10388626 6 2009-08-11 16:05:27.267 2009-08-11 16:42:45.837 10355890 10912348 10945095 40 2009-01-05 12:16:17.787 2010-01-05 14:25:35.847 8355863 10355912 10388638 42 2006-10-24 15:12:15.247 2009-08-13 11:26:25.690 80 10912347 10945094 39 2009-01-05 12:16:17.787 2010-01-04 14:21:04.847 8355863 10355911 10388637 41 2006-10-24 15:12:15.247 2009-01-13 13:47:22.177 80 10912354 10945101 5 2009-01-22 10:36:03.900 2009-10-28 17:27:14.280 8912971 5374188 5406935 1 2008-06-17 16:49:15.183 2008-06-17 16:49:15.183 5374184 5374189 5406936 2 2008-06-17 16:49:15.183 2008-06-17 16:50:38.653 5374184 11797060 11829813 1 2010-04-26 16:05:08.450 2010-04-26 16:05:08.450 11797058 3637927 3670677 1 2008-05-07 14:59:30.507 2008-05-07 14:59:30.507 3637922 3637930 3670680 2 2008-05-07 14:59:30.507 2008-05-07 15:07:56.753 3637922 11797054 11829807 47 2006-10-24 15:12:15.247 2010-01-04 17:02:19.407 80 4915259 4948022 38 2008-05-07 14:59:30.507 2008-05-30 10:48:13.507 3637922 3735607 3637922 2008-05-07 15:07:00.377 2008-05-07 15:07:00.377 298209 1 3735602 3735602 3637922 2008-05-07 15:07:00.377 2008-05-08 13:53:10.010 440368 2 4915267 4948030 39 2008-05-07 14:59:30.507 2008-06-05 13:13:29.093 3637922 10355691 10388419 12 2009-07-31 11:41:34.023 2009-08-03 00:45:38.607 10355664 10944585 10911819 11797149 11829897 41 2009-01-05 12:16:17.787 2010-01-05 14:33:23.970 8355863 11797151 11829899 42 2009-01-05 12:16:17.787 2010-04-29 10:05:17.080 8355863 238 2 10 2007-01-24 09:23:15.067 2007-06-21 09:46:15.063 11797153 11829901 43 2009-01-05 12:16:17.787 2010-04-29 11:47:17.337 8355863 11797155 11829903 44 2009-01-05 12:16:17.787 2010-04-29 11:57:06.707 8355863 24379787 0 10355664 2014-02-10 20:45:58.237 2014-02-10 20:45:58.237 11797157 11829905 45 2009-01-05 12:16:17.787 2010-04-29 11:58:11.757 8355863 10944600 10911836 4915257 4948020 43 2006-10-17 21:39:33.643 2008-05-30 10:40:48.417 10 4915255 4948018 28 2006-10-24 15:12:15.247 2008-05-30 15:54:04.350 80 10944612 10911848 13369349 13336581 4915202 4947970 57 2007-06-22 13:07:08.573 2008-06-03 13:17:05.313 1179738 4915201 4947969 56 2007-06-22 13:07:08.573 2008-05-30 11:51:34.147 1179738 4849689 3 1179738 2008-06-03 13:17:15.030 2008-06-05 12:56:15.037 4849688 2 1179738 2008-06-03 13:17:15.027 2008-06-03 13:18:15.037 24379703 0 10355664 2014-02-09 01:00:41.677 2014-02-09 01:00:41.677 4849675 7 10 2008-06-03 12:48:15.023 2008-06-09 09:39:15.023 9830450 9797681 24379733 0 10355664 2014-02-10 04:57:59.257 2014-02-10 15:13:32.080 9830458 9797689 18579773 18645288 77 2006-10-24 15:12:15.247 2011-07-15 07:39:39.473 80 18579772 18645287 32 2011-05-30 20:39:26.043 2011-06-03 15:41:37.193 17236005 4882506 2 2008-06-03 12:47:41.020 2008-06-03 12:47:41.020 4882507 2 2008-06-03 12:47:47.410 2008-06-03 12:47:47.410 3735622 3637922 2008-05-16 13:41:42.110 2008-06-09 09:36:53.140 345273 2 206 8 80 2006-12-01 15:01:15.023 2008-01-10 09:34:15.090 205 33 10 2006-12-01 15:01:15.023 2008-04-03 14:00:15.150 24379751 0 10355664 2014-02-10 10:50:08.147 2014-02-10 10:50:08.147 8093796 8061034 8093798 8061036 8093801 8061038 8093803 8061040 8093805 {noformat} h2.  ]]> 8061042 8093807 {code}\\ # Submit the Metadata to SSDS using the SSDSLoads application h2. Publishing data to SSDS ]]> 8061044 8093809 {code} Submit the Metadata to SSDS using the SSDSLoads application h2. Publishing data to SSDS]]> 8061046 1212503 1179736 8093811 {code} # Submit the Metadata to SSDS using the SSDSLoads application h2. Publishing data to SSDS]]> 8061048 13959178 13926410 1212504 1179737 8093812 {code} *# Submit the Metadata to SSDS using the SSDSLoads application h2. Establish data publishing application]]> 8061049 18 16 1 2006-10-17 21:39:33.420 2006-10-17 21:39:33.420 9 8093813 {code} # Submit the Metadata to SSDS using the SSDSLoads application h2. Establish data publishing application ]]> 8061050 1212506 SSDS_Metadata to Solstice->SSDS_Metadata to match the one used for SSDSDevPC and run. ## Startup new-ssds jboss. ## Run all MSE Urls. ## Have Mike run Stanard``Variable updates. ## Rebuild code and deploy with messaging turned back on. ## Verify all data of interest for MSE is being plotted ## Move MTM3 web apps to same form as MSE. ## New Device creation/editing page ## New Person creation/editing page ## Point old jsp's to new jsp's ## Point old Get``Original``Data``Servlet (ssds.shore and ssdpsub) to new Get``Original``Data``Servlet ## Migrate HOOVES. ## Shut off ruminate on predator. ## Shut off cimt on predator. ## Undeploy all SSDS stuff except for access.war that has forwards in it. ## Verify what Luis is using for Salinity is just Get``Original``Data``Servlet. ## Leave for a bit, but remove SSDS from solstice. ## Remove SSDS from Fog. ## Clean out do fresh installation on prey (retask). ## Turn off replication of data from tornado to ssdspub ## Remove SSDS data database from SSDSPub and all that should remain is a few .war files pointing to new-ssds. ## Ensure M0 PC02 Plots working (M0 Turnaround) # Break up deployments in metadata for Mooring into pre-deployment and deployment (M0 is the real issue) (no progress, re-iterated by Francisco) # In build, fix so that the jsf-lib directory is removed and install jstl.jar & standard.jar # What is the official MSE name? (MTM Meeting next week). # Find NetCDF Files for CIMT data and send URLs to Brian Fulfrost # Web page access to data from final ADCP netcdf file (Mike M completed) # Find location and put XML schemas (and associated DTD's out on the public web). Also include that as part of the build process (http://new-ssds.mbari.org/ssds-docs/xml/schema/SSDS_Metadata.xsd) # Add uri attribute to Standard``Variable (Valid URI) # Alternate primary key on SV (name, uri) # Can we implement a method on Process``Run``Access that can do cascade deletes? (done in new architecture on Data``Producer``DAO and is called "deepDelete") # "I'm calling Process``Run``Access.update() with additional inputs and they are not getting added. (new architecture does this) # Get the graphs on the CIMT web site working with device 1480. # The Time/Date box in the SIAM``Raw``Data``Access``Page states it is PST, but that is no longer true since we moved to having the Jboss instance running in GMT. # Implement findWithinDateRange() (Mike M) method is called findByDateRangeAndName and can be invoked with null name to just search by date. # Implement findByDataProducerGroupName() [remove findByLikeDataProducerGroupName() & other ..like.. methods] (Mike M) # Return count methods, e.g. countOfFindByDataProducerType() (Mike M) (ongoing, but done for the ones listed here) # findByLikeNameWithinTimeAndWithinGeospatialCube() [remove the ..like.. & use exactMatch boolean] (Mike M) (Done and method is called findByNameAndTimeAndGeospatialCube). # Would these countOf...() methods be easy to implement? They would sure be handy to have from a user interface perspective. (Mike M) (Done for the ones in this completed section, but still an ongoing effort). # Add Resouce``BLOB to Resource (Andrew). # Add Keywords to Data``Container through servlet (Andrew) (This should work now, but not really tested) # Query for Data``Container by Keyword (Andrew) (Done, but I am going to refactor to implement one query method like I did in Data``Producer``DAO). # Query for Data``Container by Lat/Lon and by time (Andrew) (Done, but I am going to refactor to implement one query method like I did in Data``Producer``DAO). # Object/XML Builder done for new architecture. # Metadata``Access``Servlet updated for new architecture # Fixed bug in Metadata``Factory to return primitive classes when primitive text is passed in. # Fixed DTS job to copy from old database to new on Fog (1/11/2006) # Change SSDS so it can handle new Summary``Packet from SIAM. No change was necessary as Tom is exporting Summary``Packets as Sensor``Data``Packets # Figure out if we need to submit any text for SSDS Project for Annual Report (due 2/1) # Get Andrew's Metadata``Access``Servlet documentation up on servlet and/or ssds web application # Write up new architecture and migration path for new machine and architecture deployment. # Simplify operation tasks (startup, shutdown, health, etc.) # Document requirements for data access for MSE and get those and schedule to Keith ## Talk to Charlie, Jim Barry, Bill Ussler, Rendy, Chris Lovera about how they process data from the instruments currently. ## Create list of UI goals for this (and beyond) and figure out taks/skills for that. # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## On plots, link sensor manufacturer and model and put on plot itself (if possible) ## If min/max value exist for record variables, clip generated plots to those values. Also can use the new displayMin/displayMax variables where data gaps are) # Look into Ken's suggestion about ID's for deployment. From his email {{{ All, When looking at the raw data in SSDS is can be very difficult to figure out what is what mooring and will get even more so as time goes by. This email is an attempt to get discussions going on how to deal with eliminating the confusion. All instruments, including controller cans can (and have been) be swapped during the deployment. It would be nice to have an ID for each deployment i.e. 2004 M0, 2005 M0, 2005 M2.If this ID were set up in the can as the parent ID just before deployment (i.e. aboard ship on the way out), then the problem of trying to separate out test data from deployed data would be easier to manage. It should also help resolve the issue of having latitude and longitude data in pucks. Instead the mooring ID would have the location associated with it. Regards, Ken Heller}}} # Implement Query: Find all instruments currently deployed on a "deployment" # Implement Query: They want to be able to selectively view only Metdata``Packets (Record``Type) and find only the most recent one. # Implement Query: They want a way to know what is currently deployed (user interface). # Fix Java``Script bug in SIAM Raw Data Access Page: This is a bug that Andy Hamilton showed me. When you select start to end time option, the get data, then click on back, the Javascript diables some of the time selection fields, which then invalidate the query and if you try to get the data again, you get a Null``Pointer``Exception. # Follow up with ops group with meeting on ID's, device, sensors, etc. and clean up device DB in SSDS # Add way to lookup instruments by platform on the device table # Merge access method (SIAM``Raw``Data``Access and Get``Original``Data``Servlet) to have same options and same output, web page simply forms the URL. An email{{{ Hi Kevin, I'm noticing that the GetOriginalDataServlet app seems to be adding an additional CR/LF between the ISUS samples (each sample has four data records) when I snarf it directly as opposed to displaying it via a browser. See the URL below to see what I'm describing. It won't be a problem as far as I'm concerned just something I noticed. http://www2.mbari.org/coletti/m0_isus.cgi Luke And another Ah Ha, look what happens when you set noHTMLHeader=0 (URL below)... http://ssds.shore.mbari.org:8080/access/GetOriginalDataServlet?platformID=1299&metadataID=0 &recordTypeID=1&isi=0&noHTMLHeader=0&deviceID=1268&startDateTime=20040603.223400&endDateTime=20040606.223400, Luke }}} # Implmement basic metadata/data QC internal to SSDS (watchdog) # Refactor Ruminate in new architecture # When information in XML is inconsistent (device ID) with that of packet, somebody should be notified. # If device comes in with new ID, it will be updated/persisted, should not' # Performance investigation of services (sort of ongoing process) # The XML``Metadata``Tracker does not look like it is working properly (especially on AUV files) in the ruminate process # Andrew metioned that the depths (Data``Container) are rounding. # Boolean not setting on incoming Metadata``Access``Servlet query (verify this is fixed in new architecture, then move to completed) # Connect up raw stream access to the SQL database stream storage # Add parameter that the user can ask for the last X number of records. # Can we setup the Record``Description so that it can define a regular expression that can parse the whole record into variables? # Can we have a service that will automatically take NEMA strings from GPS's and convert them to a more useful format? # Implement Time``Indexed``Net``CDF``Access class # Crawl all data stream files and republish data to SQL topic to populate DB with device stream data. # Get NetCDF Watchdog going to keep NetCDF files created for those Data``Containers that ask for them. # Get XML Defined for the Nortek binary case (variable binary length) # Data servlet variables out of order and no flag for units # Fix Bob's Code so it queries by deployment name, not device name then change 1305 and 1414 device names back to OASIS Bouy.Then go back and clean up device list per Paul C. (Remove some of the M1/M2 designations). Remove devices 1307 and 1308 from database. # Re-run all loads for AUVCTD data onto the production machine to make sure it is all there(?) # When SSDSPUB tries to read from dods.mbari.org, it times out. h3. Future Tasks (Shelved) # Give Nancy Barr a base URL and she can have one of her crawlers check the links # Look at porting code to J2SE 5 (Generics, Iterators). # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Develop method for handling app deployment on heckel/jeckel # Can we use Bob Herlien to do some development work? # Get Mike M and Dave from (CenCOOS) and talk through SSDS for CenCOOS and getting them AUV data from MBARI. # Apply AJAX to web pages where performance problems exist ([http://code.jalenack.com/periodic/ Example]) # Fix Metadata links on CIMT pages to make them dynamic (or replace with different view, not XML) # Pull the MBARI ADCP page off the CIMT site # Look at [http://www.sensescape.com/scivis/index.html Visualization in Marine Science] # Look into [http://www.openmi.org OpenMI] # Turn on wrapper generator unit tests in continuous integration and process/display results # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## Make any direction plot (wind, heading, etc.) plot as points, not lines ## Put nominal lattitude and longitude in plot titles ## Have capability to turn on/off autoscale on plots and specify range # Get UI Design meetings going with the ops group and get portal prototype going # Ken Heller came by to ask to turn on more batch created plots for CIMT # Put link to ACE jar in web application # Ken also had another request to be able to see all variables plotted as thumbnails on one page, be able to select various plots and then have them show as full size together on another page (a merging of sorts for data). # Add pages for metadata editing (MetadataEditingNote) # SIAM``Raw``Data``Access Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement Query: Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location). This is specifically to support uploading the JARS for SIAM and linking them to Devices and Data``Producers(Deployments). # Implement Query: They would like to be able to store the driver class name that is linked to a Device so that they can have the user simply choose a device and it will start with that driver name. # Instead of using comand line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into having SSDS create "README" type files in the same location as certain Data``Containers (This could be a perl script bot) # Hold Brown Bag on architecture and uses. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional Standard``Variables, Standard``Units, Standard``Keywords, Standard``Domain, Standard``Reference``Scale, Device``Type, Resource``Type, Data``Producer``Group, Data``Container``Group and then notify the user of that change so they can change their source. # Remove Deployment info from PUCK XML (someday, probably for MSE). # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. # Write tests for Resource``BLOB->Object``Builder for byte array and verify that it is working correctly. # Send API's for SSDS ingest and Data``Stream Access to Data``Stream LOOKING group # Look at Mike Godin's Schema changes for AOSN # Can I embed the business logic documentation as Java``Doc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # See if I can use NT logins with Merlia driver in SSDS # Have the capability to turn off emails when resubmit is done. # Implmement generic way of sending data/XML ## Create application that reads in NetCDF file and generates XML to match our schema # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Cut down on JAR file bloat (happening in new arch) # Crawl all the old OASIS files and load the metadata into SSDS. # Meet with MMI to talk about pulling our SV and SU from ontologies. # XML with schemas declared not validating # Add mechanism that users can drop files of known type with all metadata in known format into a directory and SSDS will automatically register and store it. # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # Implement ingest socket -> JMS topic # Add QC flag to SSDS packets and match in database (SSDS_Data). # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # In Packet``Output``Manager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and Packet``Outputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add end of line terminator as separator in parsing packet records (not files) # Can I create something that will emulate reading from a serial port but is back by the SSDS data service? This would be done so that native applications that are used to reading from serial ports, can still do so. To be honest, this would be really cool, but seems fragile and unlikely. This started with the Nortek, so look at Nortek.no for what they called "Online applications" # Get LOBO data into SSDS # Get raw data porting from SSDS to LAS server working for Mooring data (Leverage Mike G's stuff). # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" data by storing comments with the data and returning that to user's upon request (think NetCDFs history) # Edit bad packets that are serialized to disk # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data) # Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Setup service that will calculate salinity automatically. # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Query: "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Create Process to create useable data file from inductive CTD string # Look at integrating Earth Google using services. # HOOVES Improvements: ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface # Look at NPS AUV Mission Planner as a way to interact with AUV data.Don Brutzman and Peter Flynn came to give a demo on this and it seems to be a possibility for a tool that can integrate and visualize modeling and collected data (as well as vehicle tracks). Just check Don's web address to find information ]]> 1179739 8093815 {code} # Submit the Metadata to SSDS using the SSDSLoads application h2. Establish data publishing application]]> 8061052 8093816 {code} # Submit the Metadata to SSDS using the SSDSLoads application h2. Establish data publishing application]]> 8061053 1212502 1179735 8093818 {code} # Submit the Metadata to SSDS using the SSDSLoads application h2. Establish data publishing application]]> 8061055 8093820 {code} # We recommend checking in the final XML to the puckxml module in MBARI's CVS. # Submit the Metadata to SSDS using the SSDSLoads application (available at [http://new-ssds.mbari.org/ssds-docs/client/]. The \-h option provides a usage note): {code} java -jar ssdsLoads-new-ssds.jar -d 1696.xml {code} h2. Establish data publishing application]]> 8061057 8093822 {code} # We recommend checking in the final XML to the puckxml module in MBARI's CVS. # Submit the Metadata to SSDS using the SSDSLoads application (available at [http://new-ssds.mbari.org/ssds-docs/client/]. The \-h option provides a usage note): {code} java -jar ssdsLoads-new-ssds.jar -d 1696.xml {code} ## If a mistake is made in the metadata (e.g. forgetting the platform Deployment) it may be easier to undo the submission and start over - do a deep delete on the parent deployment if this is the case. ## Minor attribute fixes may be done by editing the database once the deployments have been loaded h2. Establish data publishing application]]> 8061059 1212507 SSDS_Metadata to Solstice->SSDS_Metadata to match the one used for SSDSDevPC and run. ## Startup new-ssds jboss. ## Run all MSE Urls. ## Have Mike run Stanard``Variable updates. ## Rebuild code and deploy with messaging turned back on. ## Verify all data of interest for MSE is being plotted ## Move MTM3 web apps to same form as MSE. ## New Device creation/editing page ## New Person creation/editing page ## Point old jsp's to new jsp's ## Point old Get``Original``Data``Servlet (ssds.shore and ssdpsub) to new Get``Original``Data``Servlet ## Migrate HOOVES. ## Shut off ruminate on predator. ## Shut off cimt on predator. ## Undeploy all SSDS stuff except for access.war that has forwards in it. ## Verify what Luis is using for Salinity is just Get``Original``Data``Servlet. ## Leave for a bit, but remove SSDS from solstice. ## Remove SSDS from Fog. ## Clean out do fresh installation on prey (retask). ## Turn off replication of data from tornado to ssdspub ## Remove SSDS data database from SSDSPub and all that should remain is a few .war files pointing to new-ssds. ## Ensure M0 PC02 Plots working (M0 Turnaround) # Break up deployments in metadata for Mooring into pre-deployment and deployment (M0 is the real issue) (no progress, re-iterated by Francisco) # In build, fix so that the jsf-lib directory is removed and install jstl.jar & standard.jar # What is the official MSE name? (MTM Meeting next week). # Find NetCDF Files for CIMT data and send URLs to Brian Fulfrost # Web page access to data from final ADCP netcdf file (Mike M completed) # Find location and put XML schemas (and associated DTD's out on the public web). Also include that as part of the build process (http://new-ssds.mbari.org/ssds-docs/xml/schema/SSDS_Metadata.xsd) # Add uri attribute to Standard``Variable (Valid URI) # Alternate primary key on SV (name, uri) # Can we implement a method on Process``Run``Access that can do cascade deletes? (done in new architecture on Data``Producer``DAO and is called "deepDelete") # "I'm calling Process``Run``Access.update() with additional inputs and they are not getting added. (new architecture does this) # Get the graphs on the CIMT web site working with device 1480. # The Time/Date box in the SIAM``Raw``Data``Access``Page states it is PST, but that is no longer true since we moved to having the Jboss instance running in GMT. # Implement findWithinDateRange() (Mike M) method is called findByDateRangeAndName and can be invoked with null name to just search by date. # Implement findByDataProducerGroupName() [remove findByLikeDataProducerGroupName() & other ..like.. methods] (Mike M) # Return count methods, e.g. countOfFindByDataProducerType() (Mike M) (ongoing, but done for the ones listed here) # findByLikeNameWithinTimeAndWithinGeospatialCube() [remove the ..like.. & use exactMatch boolean] (Mike M) (Done and method is called findByNameAndTimeAndGeospatialCube). # Would these countOf...() methods be easy to implement? They would sure be handy to have from a user interface perspective. (Mike M) (Done for the ones in this completed section, but still an ongoing effort). # Add Resouce``BLOB to Resource (Andrew). # Add Keywords to Data``Container through servlet (Andrew) (This should work now, but not really tested) # Query for Data``Container by Keyword (Andrew) (Done, but I am going to refactor to implement one query method like I did in Data``Producer``DAO). # Query for Data``Container by Lat/Lon and by time (Andrew) (Done, but I am going to refactor to implement one query method like I did in Data``Producer``DAO). # Object/XML Builder done for new architecture. # Metadata``Access``Servlet updated for new architecture # Fixed bug in Metadata``Factory to return primitive classes when primitive text is passed in. # Fixed DTS job to copy from old database to new on Fog (1/11/2006) # Change SSDS so it can handle new Summary``Packet from SIAM. No change was necessary as Tom is exporting Summary``Packets as Sensor``Data``Packets # Figure out if we need to submit any text for SSDS Project for Annual Report (due 2/1) # Get Andrew's Metadata``Access``Servlet documentation up on servlet and/or ssds web application # Write up new architecture and migration path for new machine and architecture deployment. # Simplify operation tasks (startup, shutdown, health, etc.) # Document requirements for data access for MSE and get those and schedule to Keith ## Talk to Charlie, Jim Barry, Bill Ussler, Rendy, Chris Lovera about how they process data from the instruments currently. ## Create list of UI goals for this (and beyond) and figure out taks/skills for that. # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## On plots, link sensor manufacturer and model and put on plot itself (if possible) ## If min/max value exist for record variables, clip generated plots to those values. Also can use the new displayMin/displayMax variables where data gaps are) # Look into Ken's suggestion about ID's for deployment. From his email {noformat} All, When looking at the raw data in SSDS is can be very difficult to figure out what is what mooring and will get even more so as time goes by. This email is an attempt to get discussions going on how to deal with eliminating the confusion. All instruments, including controller cans can (and have been) be swapped during the deployment. It would be nice to have an ID for each deployment i.e. 2004 M0, 2005 M0, 2005 M2.If this ID were set up in the can as the parent ID just before deployment (i.e. aboard ship on the way out), then the problem of trying to separate out test data from deployed data would be easier to manage. It should also help resolve the issue of having latitude and longitude data in pucks. Instead the mooring ID would have the location associated with it. Regards, Ken Heller {noformat} # Implement Query: Find all instruments currently deployed on a "deployment" # Implement Query: They want to be able to selectively view only Metdata``Packets (Record``Type) and find only the most recent one. # Implement Query: They want a way to know what is currently deployed (user interface). # Fix Java``Script bug in SIAM Raw Data Access Page: This is a bug that Andy Hamilton showed me. When you select start to end time option, the get data, then click on back, the Javascript diables some of the time selection fields, which then invalidate the query and if you try to get the data again, you get a Null``Pointer``Exception. # Follow up with ops group with meeting on ID's, device, sensors, etc. and clean up device DB in SSDS # Add way to lookup instruments by platform on the device table # Merge access method (SIAM``Raw``Data``Access and Get``Original``Data``Servlet) to have same options and same output, web page simply forms the URL. An email {noformat} Hi Kevin, I'm noticing that the GetOriginalDataServlet app seems to be adding an additional CR/LF between the ISUS samples (each sample has four data records) when I snarf it directly as opposed to displaying it via a browser. See the URL below to see what I'm describing. It won't be a problem as far as I'm concerned just something I noticed. http://www2.mbari.org/coletti/m0_isus.cgi Luke And another Ah Ha, look what happens when you set noHTMLHeader=0 (URL below)... http://ssds.shore.mbari.org:8080/access/GetOriginalDataServlet?platformID=1299&metadataID=0 &recordTypeID=1&isi=0&noHTMLHeader=0&deviceID=1268&startDateTime=20040603.223400&endDateTime=20040606.223400, Luke {noformat} # Implmement basic metadata/data QC internal to SSDS (watchdog) # Refactor Ruminate in new architecture # When information in XML is inconsistent (device ID) with that of packet, somebody should be notified. # If device comes in with new ID, it will be updated/persisted, should not' # Performance investigation of services (sort of ongoing process) # The XML``Metadata``Tracker does not look like it is working properly (especially on AUV files) in the ruminate process # Andrew metioned that the depths (Data``Container) are rounding. # Boolean not setting on incoming Metadata``Access``Servlet query (verify this is fixed in new architecture, then move to completed) # Connect up raw stream access to the SQL database stream storage # Add parameter that the user can ask for the last X number of records. # Can we setup the Record``Description so that it can define a regular expression that can parse the whole record into variables? # Can we have a service that will automatically take NEMA strings from GPS's and convert them to a more useful format? # Implement Time``Indexed``Net``CDF``Access class # Crawl all data stream files and republish data to SQL topic to populate DB with device stream data. # Get NetCDF Watchdog going to keep NetCDF files created for those Data``Containers that ask for them. # Get XML Defined for the Nortek binary case (variable binary length) # Data servlet variables out of order and no flag for units # Fix Bob's Code so it queries by deployment name, not device name then change 1305 and 1414 device names back to OASIS Bouy.Then go back and clean up device list per Paul C. (Remove some of the M1/M2 designations). Remove devices 1307 and 1308 from database. # Re-run all loads for AUVCTD data onto the production machine to make sure it is all there(?) # When SSDSPUB tries to read from dods.mbari.org, it times out. h3. Future Tasks (Shelved) # Give Nancy Barr a base URL and she can have one of her crawlers check the links # Look at porting code to J2SE 5 (Generics, Iterators). # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Develop method for handling app deployment on heckel/jeckel # Can we use Bob Herlien to do some development work? # Get Mike M and Dave from (CenCOOS) and talk through SSDS for CenCOOS and getting them AUV data from MBARI. # Apply AJAX to web pages where performance problems exist ([http://code.jalenack.com/periodic/ Example]) # Fix Metadata links on CIMT pages to make them dynamic (or replace with different view, not XML) # Pull the MBARI ADCP page off the CIMT site # Look at [http://www.sensescape.com/scivis/index.html Visualization in Marine Science] # Look into [http://www.openmi.org OpenMI] # Turn on wrapper generator unit tests in continuous integration and process/display results # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## Make any direction plot (wind, heading, etc.) plot as points, not lines ## Put nominal lattitude and longitude in plot titles ## Have capability to turn on/off autoscale on plots and specify range # Get UI Design meetings going with the ops group and get portal prototype going # Ken Heller came by to ask to turn on more batch created plots for CIMT # Put link to ACE jar in web application # Ken also had another request to be able to see all variables plotted as thumbnails on one page, be able to select various plots and then have them show as full size together on another page (a merging of sorts for data). # Add pages for metadata editing (MetadataEditingNote) # SIAM``Raw``Data``Access Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement Query: Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location). This is specifically to support uploading the JARS for SIAM and linking them to Devices and Data``Producers(Deployments). # Implement Query: They would like to be able to store the driver class name that is linked to a Device so that they can have the user simply choose a device and it will start with that driver name. # Instead of using comand line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into having SSDS create "README" type files in the same location as certain Data``Containers (This could be a perl script bot) # Hold Brown Bag on architecture and uses. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional Standard``Variables, Standard``Units, Standard``Keywords, Standard``Domain, Standard``Reference``Scale, Device``Type, Resource``Type, Data``Producer``Group, Data``Container``Group and then notify the user of that change so they can change their source. # Remove Deployment info from PUCK XML (someday, probably for MSE). # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. # Write tests for Resource``BLOB->Object``Builder for byte array and verify that it is working correctly. # Send API's for SSDS ingest and Data``Stream Access to Data``Stream LOOKING group # Look at Mike Godin's Schema changes for AOSN # Can I embed the business logic documentation as Java``Doc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # See if I can use NT logins with Merlia driver in SSDS # Have the capability to turn off emails when resubmit is done. # Implmement generic way of sending data/XML ## Create application that reads in NetCDF file and generates XML to match our schema # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Cut down on JAR file bloat (happening in new arch) # Crawl all the old OASIS files and load the metadata into SSDS. # Meet with MMI to talk about pulling our SV and SU from ontologies. # XML with schemas declared not validating # Add mechanism that users can drop files of known type with all metadata in known format into a directory and SSDS will automatically register and store it. # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # Implement ingest socket -> JMS topic # Add QC flag to SSDS packets and match in database (SSDS_Data). # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # In Packet``Output``Manager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and Packet``Outputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add end of line terminator as separator in parsing packet records (not files) # Can I create something that will emulate reading from a serial port but is back by the SSDS data service? This would be done so that native applications that are used to reading from serial ports, can still do so. To be honest, this would be really cool, but seems fragile and unlikely. This started with the Nortek, so look at Nortek.no for what they called "Online applications" # Get LOBO data into SSDS # Get raw data porting from SSDS to LAS server working for Mooring data (Leverage Mike G's stuff). # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" data by storing comments with the data and returning that to user's upon request (think NetCDFs history) # Edit bad packets that are serialized to disk # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data) # Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Setup service that will calculate salinity automatically. # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Query: "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Create Process to create useable data file from inductive CTD string # Look at integrating Earth Google using services. # HOOVES Improvements: ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface # Look at NPS AUV Mission Planner as a way to interact with AUV data.Don Brutzman and Peter Flynn came to give a demo on this and it seems to be a possibility for a tool that can integrate and visualize modeling and collected data (as well as vehicle tracks). Just check Don's web address to find information ]]> 1179740 1212508 SSDS_Metadata to Solstice->SSDS_Metadata to match the one used for SSDSDevPC and run. ## Startup new-ssds jboss. ## Run all MSE Urls. ## Have Mike run Stanard``Variable updates. ## Rebuild code and deploy with messaging turned back on. ## Verify all data of interest for MSE is being plotted ## Move MTM3 web apps to same form as MSE. ## New Device creation/editing page ## New Person creation/editing page ## Point old jsp's to new jsp's ## Point old Get``Original``Data``Servlet (ssds.shore and ssdpsub) to new Get``Original``Data``Servlet ## Migrate HOOVES. ## Shut off ruminate on predator. ## Shut off cimt on predator. ## Undeploy all SSDS stuff except for access.war that has forwards in it. ## Verify what Luis is using for Salinity is just Get``Original``Data``Servlet. ## Leave for a bit, but remove SSDS from solstice. ## Remove SSDS from Fog. ## Clean out do fresh installation on prey (retask). ## Turn off replication of data from tornado to ssdspub ## Remove SSDS data database from SSDSPub and all that should remain is a few .war files pointing to new-ssds. ## Ensure M0 PC02 Plots working (M0 Turnaround) # Break up deployments in metadata for Mooring into pre-deployment and deployment (M0 is the real issue) (no progress, re-iterated by Francisco) # In build, fix so that the jsf-lib directory is removed and install jstl.jar & standard.jar # What is the official MSE name? (MTM Meeting next week). # Find NetCDF Files for CIMT data and send URLs to Brian Fulfrost # Web page access to data from final ADCP netcdf file (Mike M completed) # Find location and put XML schemas (and associated DTD's out on the public web). Also include that as part of the build process (http://new-ssds.mbari.org/ssds-docs/xml/schema/SSDS_Metadata.xsd) # Add uri attribute to Standard``Variable (Valid URI) # Alternate primary key on SV (name, uri) # Can we implement a method on Process``Run``Access that can do cascade deletes? (done in new architecture on Data``Producer``DAO and is called "deepDelete") # "I'm calling Process``Run``Access.update() with additional inputs and they are not getting added. (new architecture does this) # Get the graphs on the CIMT web site working with device 1480. # The Time/Date box in the SIAM``Raw``Data``Access``Page states it is PST, but that is no longer true since we moved to having the Jboss instance running in GMT. # Implement findWithinDateRange() (Mike M) method is called findByDateRangeAndName and can be invoked with null name to just search by date. # Implement findByDataProducerGroupName() [remove findByLikeDataProducerGroupName() & other ..like.. methods] (Mike M) # Return count methods, e.g. countOfFindByDataProducerType() (Mike M) (ongoing, but done for the ones listed here) # findByLikeNameWithinTimeAndWithinGeospatialCube() [remove the ..like.. & use exactMatch boolean] (Mike M) (Done and method is called findByNameAndTimeAndGeospatialCube). # Would these countOf...() methods be easy to implement? They would sure be handy to have from a user interface perspective. (Mike M) (Done for the ones in this completed section, but still an ongoing effort). # Add Resouce``BLOB to Resource (Andrew). # Add Keywords to Data``Container through servlet (Andrew) (This should work now, but not really tested) # Query for Data``Container by Keyword (Andrew) (Done, but I am going to refactor to implement one query method like I did in Data``Producer``DAO). # Query for Data``Container by Lat/Lon and by time (Andrew) (Done, but I am going to refactor to implement one query method like I did in Data``Producer``DAO). # Object/XML Builder done for new architecture. # Metadata``Access``Servlet updated for new architecture # Fixed bug in Metadata``Factory to return primitive classes when primitive text is passed in. # Fixed DTS job to copy from old database to new on Fog (1/11/2006) # Change SSDS so it can handle new Summary``Packet from SIAM. No change was necessary as Tom is exporting Summary``Packets as Sensor``Data``Packets # Figure out if we need to submit any text for SSDS Project for Annual Report (due 2/1) # Get Andrew's Metadata``Access``Servlet documentation up on servlet and/or ssds web application # Write up new architecture and migration path for new machine and architecture deployment. # Simplify operation tasks (startup, shutdown, health, etc.) # Document requirements for data access for MSE and get those and schedule to Keith ## Talk to Charlie, Jim Barry, Bill Ussler, Rendy, Chris Lovera about how they process data from the instruments currently. ## Create list of UI goals for this (and beyond) and figure out taks/skills for that. # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## On plots, link sensor manufacturer and model and put on plot itself (if possible) ## If min/max value exist for record variables, clip generated plots to those values. Also can use the new displayMin/displayMax variables where data gaps are) # Look into Ken's suggestion about ID's for deployment. From his email {noformat} All, When looking at the raw data in SSDS is can be very difficult to figure out what is what mooring and will get even more so as time goes by. This email is an attempt to get discussions going on how to deal with eliminating the confusion. All instruments, including controller cans can (and have been) be swapped during the deployment. It would be nice to have an ID for each deployment i.e. 2004 M0, 2005 M0, 2005 M2.If this ID were set up in the can as the parent ID just before deployment (i.e. aboard ship on the way out), then the problem of trying to separate out test data from deployed data would be easier to manage. It should also help resolve the issue of having latitude and longitude data in pucks. Instead the mooring ID would have the location associated with it. Regards, Ken Heller {noformat} # Implement Query: Find all instruments currently deployed on a "deployment" # Implement Query: They want to be able to selectively view only Metdata``Packets (Record``Type) and find only the most recent one. # Implement Query: They want a way to know what is currently deployed (user interface). # Fix Java``Script bug in SIAM Raw Data Access Page: This is a bug that Andy Hamilton showed me. When you select start to end time option, the get data, then click on back, the Javascript diables some of the time selection fields, which then invalidate the query and if you try to get the data again, you get a Null``Pointer``Exception. # Follow up with ops group with meeting on ID's, device, sensors, etc. and clean up device DB in SSDS # Add way to lookup instruments by platform on the device table # Merge access method (SIAM``Raw``Data``Access and Get``Original``Data``Servlet) to have same options and same output, web page simply forms the URL. An email {noformat} Hi Kevin, I'm noticing that the GetOriginalDataServlet app seems to be adding an additional CR/LF between the ISUS samples (each sample has four data records) when I snarf it directly as opposed to displaying it via a browser. See the URL below to see what I'm describing. It won't be a problem as far as I'm concerned just something I noticed. http://www2.mbari.org/coletti/m0_isus.cgi Luke And another Ah Ha, look what happens when you set noHTMLHeader=0 (URL below)... http://ssds.shore.mbari.org:8080/access/GetOriginalDataServlet?platformID=1299&metadataID=0 &recordTypeID=1&isi=0&noHTMLHeader=0&deviceID=1268&startDateTime=20040603.223400&endDateTime=20040606.223400, Luke {noformat} # Implmement basic metadata/data QC internal to SSDS (watchdog) # Refactor Ruminate in new architecture # When information in XML is inconsistent (device ID) with that of packet, somebody should be notified. # If device comes in with new ID, it will be updated/persisted, should not' # Performance investigation of services (sort of ongoing process) # The XML``Metadata``Tracker does not look like it is working properly (especially on AUV files) in the ruminate process # Andrew metioned that the depths (Data``Container) are rounding. # Boolean not setting on incoming Metadata``Access``Servlet query (verify this is fixed in new architecture, then move to completed) # Connect up raw stream access to the SQL database stream storage # Add parameter that the user can ask for the last X number of records. # Can we setup the Record``Description so that it can define a regular expression that can parse the whole record into variables? # Can we have a service that will automatically take NEMA strings from GPS's and convert them to a more useful format? # Implement Time``Indexed``Net``CDF``Access class # Crawl all data stream files and republish data to SQL topic to populate DB with device stream data. # Get NetCDF Watchdog going to keep NetCDF files created for those Data``Containers that ask for them. # Get XML Defined for the Nortek binary case (variable binary length) # Data servlet variables out of order and no flag for units # Fix Bob's Code so it queries by deployment name, not device name then change 1305 and 1414 device names back to OASIS Bouy.Then go back and clean up device list per Paul C. (Remove some of the M1/M2 designations). Remove devices 1307 and 1308 from database. # Re-run all loads for AUVCTD data onto the production machine to make sure it is all there(?) # When SSDSPUB tries to read from dods.mbari.org, it times out. h3. Future Tasks (Shelved) # Give Nancy Barr a base URL and she can have one of her crawlers check the links # Look at porting code to J2SE 5 (Generics, Iterators). # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Develop method for handling app deployment on heckel/jeckel # Can we use Bob Herlien to do some development work? # Get Mike M and Dave from (CenCOOS) and talk through SSDS for CenCOOS and getting them AUV data from MBARI. # Apply AJAX to web pages where performance problems exist ([http://code.jalenack.com/periodic/ Example]) # Fix Metadata links on CIMT pages to make them dynamic (or replace with different view, not XML) # Pull the MBARI ADCP page off the CIMT site # Look at [http://www.sensescape.com/scivis/index.html Visualization in Marine Science] # Look into [http://www.openmi.org OpenMI] # Turn on wrapper generator unit tests in continuous integration and process/display results # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## Make any direction plot (wind, heading, etc.) plot as points, not lines ## Put nominal lattitude and longitude in plot titles ## Have capability to turn on/off autoscale on plots and specify range # Get UI Design meetings going with the ops group and get portal prototype going # Ken Heller came by to ask to turn on more batch created plots for CIMT # Put link to ACE jar in web application # Ken also had another request to be able to see all variables plotted as thumbnails on one page, be able to select various plots and then have them show as full size together on another page (a merging of sorts for data). # Add pages for metadata editing (MetadataEditingNote) # SIAM``Raw``Data``Access Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement Query: Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location). This is specifically to support uploading the JARS for SIAM and linking them to Devices and Data``Producers(Deployments). # Implement Query: They would like to be able to store the driver class name that is linked to a Device so that they can have the user simply choose a device and it will start with that driver name. # Instead of using comand line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into having SSDS create "README" type files in the same location as certain Data``Containers (This could be a perl script bot) # Hold Brown Bag on architecture and uses. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional Standard``Variables, Standard``Units, Standard``Keywords, Standard``Domain, Standard``Reference``Scale, Device``Type, Resource``Type, Data``Producer``Group, Data``Container``Group and then notify the user of that change so they can change their source. # Remove Deployment info from PUCK XML (someday, probably for MSE). # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. # Write tests for Resource``BLOB->Object``Builder for byte array and verify that it is working correctly. # Send API's for SSDS ingest and Data``Stream Access to Data``Stream LOOKING group # Look at Mike Godin's Schema changes for AOSN # Can I embed the business logic documentation as Java``Doc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # See if I can use NT logins with Merlia driver in SSDS # Have the capability to turn off emails when resubmit is done. # Implmement generic way of sending data/XML ## Create application that reads in NetCDF file and generates XML to match our schema # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Cut down on JAR file bloat (happening in new arch) # Crawl all the old OASIS files and load the metadata into SSDS. # Meet with MMI to talk about pulling our SV and SU from ontologies. # XML with schemas declared not validating # Add mechanism that users can drop files of known type with all metadata in known format into a directory and SSDS will automatically register and store it. # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # Implement ingest socket -> JMS topic # Add QC flag to SSDS packets and match in database (SSDS_Data). # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # In Packet``Output``Manager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and Packet``Outputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add end of line terminator as separator in parsing packet records (not files) # Can I create something that will emulate reading from a serial port but is back by the SSDS data service? This would be done so that native applications that are used to reading from serial ports, can still do so. To be honest, this would be really cool, but seems fragile and unlikely. This started with the Nortek, so look at Nortek.no for what they called "Online applications" # Get LOBO data into SSDS # Get raw data porting from SSDS to LAS server working for Mooring data (Leverage Mike G's stuff). # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" data by storing comments with the data and returning that to user's upon request (think NetCDFs history) # Edit bad packets that are serialized to disk # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data) # Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Setup service that will calculate salinity automatically. # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Query: "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Create Process to create useable data file from inductive CTD string # Look at integrating Earth Google using services. # HOOVES Improvements: ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface # Look at NPS AUV Mission Planner as a way to interact with AUV data.Don Brutzman and Peter Flynn came to give a demo on this and it seems to be a possibility for a tool that can integrate and visualize modeling and collected data (as well as vehicle tracks). Just check Don's web address to find information ]]> 1179741 8093824 {code} # It's recommended that the final XML is to be checked into the puckxml module in MBARI's CVS on moonjelly. # Submit the Metadata to SSDS using the SSDSLoads application (available at [http://new-ssds.mbari.org/ssds-docs/client/]. The \-h option provides a usage note): {code} java -jar ssdsLoads-new-ssds.jar -d 1696.xml {code} ## If a mistake is made in the metadata (e.g. forgetting the platform Deployment) it may be easier to undo the submission and start over - do a deep delete on the parent deployment if this is the case. ## Minor attribute fixes may be done by editing the database once the deployments have been loaded h2. Establish data publishing application # This step assumes that there will be some application reading data from the deployed instrument. Perhaps it is a legacy application which reads the data to load into a custom data storage or visualization system. For this application to publish data to SSDS it must have visibility of each record the instrument produces as that record needs to be "packaged" into a SensorDataPacket and "published" to SSDS. # An example Java application that will package and publish a record is below: # {code} import java.io.IOException; import moos.ssds.jms.PublisherComponent; import moos.ssds.transmogrify.SSDSDevicePacket; /** *

* Publish instrument data records to the SSDS database. The client must provide * SSDS device ID, timeStamp, sequence number, and payload. *

*
* * @author : $Author: mccann $ * @version : $Revision: 1.17.2.7 $ *
*

* The * Monterey Bay Aquarium Research Institute (MBARI) provides this * documentation and code "as is", with no warranty, express * or implied, of its quality or consistency. It is provided without * support and without obligation on the part of MBARI to assist in its * use, correction, modification, or enhancement. This information * should not be published or distributed to third parties without * specific written permission from MBARI. *

*
* Copyright 2008 MBARI.
* MBARI Proprietary Information. All rights reserved.
*
*
*/ /** * @author mccann * */ public class SsdsPublisher { /** * Publish data record from an instrument to SSDS * * @param deviceID * is the SSDS Device ID for the instrument that produces the data in payload * @param epochMilliseconds * time of payload sample in milliseconds since 1/1/1970 0000 GMT * @param sequenceNumber * an incrementing number for each packet * @param payload * data from instrument */ public static void publish(long deviceID, long epochMilliseconds, long sequenceNumber, String payload) { // Create a publisher component PublisherComponent pc = new PublisherComponent(); // Create a new SensorDataPacket SSDSDevicePacket packetToSend = new SSDSDevicePacket(deviceID, payload .getBytes().length); // Set the packet type to data (0 = Metadata, 1 = Data, 2 = Message) packetToSend.setPacketType(1); // Assign the time packetToSend.setSystemTime(epochMilliseconds); // Set the parentID to 0 for a parentless deployment packetToSend.setPlatformID(0); // Set the metadataref number to 0, if the data format changes and we can // refer to a different metadata packet then this number will change. packetToSend.setMetadataRef(0); // Set the sequence number packetToSend.setSequenceNo(sequenceNumber); // Set the payload packetToSend.setDataBuffer(payload.getBytes()); // Set the record type to 1 as this is the most common situation packetToSend.setRecordType(1); try { pc.publishBytes(SSDSDevicePacket .convertToPublishableByteArray(packetToSend)); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } /** * Test of SsdsPublisher.publish() * * @param args * Ê */ public static void main(String[] args) { /* * Example packet for EITS instrument */ long eitsDeviceID = 1696; // 1696 is actual SSDS deviceID for EITS long sampleTime = 1228431283209L; // Sample time: 2008-12-04 22:54:43 long sampleSequenceNumber = 1; // May set it to ID from DB insert // Payload values must match the RecordVariables in SSDS String samplePayload = "12.2, 4.3, 768.2, 2.3, 1.2, 0.2"; // Temp, Cond, // Pres, U, // V, W SsdsPublisher.publish(eitsDeviceID, sampleTime, sampleSequenceNumber, samplePayload); } } {code}]]> 8061061 1212509 SSDS_Metadata to Solstice->SSDS_Metadata to match the one used for SSDSDevPC and run. ## Startup new-ssds jboss. ## Run all MSE Urls. ## Have Mike run Stanard``Variable updates. ## Rebuild code and deploy with messaging turned back on. ## Verify all data of interest for MSE is being plotted ## Move MTM3 web apps to same form as MSE. ## New Device creation/editing page ## New Person creation/editing page ## Point old jsp's to new jsp's ## Point old Get``Original``Data``Servlet (ssds.shore and ssdpsub) to new Get``Original``Data``Servlet ## Migrate HOOVES. ## Shut off ruminate on predator. ## Shut off cimt on predator. ## Undeploy all SSDS stuff except for access.war that has forwards in it. ## Verify what Luis is using for Salinity is just Get``Original``Data``Servlet. ## Leave for a bit, but remove SSDS from solstice. ## Remove SSDS from Fog. ## Clean out do fresh installation on prey (retask). ## Turn off replication of data from tornado to ssdspub ## Remove SSDS data database from SSDSPub and all that should remain is a few .war files pointing to new-ssds. ## Ensure M0 PC02 Plots working (M0 Turnaround) # Break up deployments in metadata for Mooring into pre-deployment and deployment (M0 is the real issue) (no progress, re-iterated by Francisco) # In build, fix so that the jsf-lib directory is removed and install jstl.jar & standard.jar # What is the official MSE name? (MTM Meeting next week). # Find NetCDF Files for CIMT data and send URLs to Brian Fulfrost # Web page access to data from final ADCP netcdf file (Mike M completed) # Find location and put XML schemas (and associated DTD's out on the public web). Also include that as part of the build process (http://new-ssds.mbari.org/ssds-docs/xml/schema/SSDS_Metadata.xsd) # Add uri attribute to Standard``Variable (Valid URI) # Alternate primary key on SV (name, uri) # Can we implement a method on Process``Run``Access that can do cascade deletes? (done in new architecture on Data``Producer``DAO and is called "deepDelete") # "I'm calling Process``Run``Access.update() with additional inputs and they are not getting added. (new architecture does this) # Get the graphs on the CIMT web site working with device 1480. # The Time/Date box in the SIAM``Raw``Data``Access``Page states it is PST, but that is no longer true since we moved to having the Jboss instance running in GMT. # Implement findWithinDateRange() (Mike M) method is called findByDateRangeAndName and can be invoked with null name to just search by date. # Implement findByDataProducerGroupName() [remove findByLikeDataProducerGroupName() & other ..like.. methods] (Mike M) # Return count methods, e.g. countOfFindByDataProducerType() (Mike M) (ongoing, but done for the ones listed here) # findByLikeNameWithinTimeAndWithinGeospatialCube() [remove the ..like.. & use exactMatch boolean] (Mike M) (Done and method is called findByNameAndTimeAndGeospatialCube). # Would these countOf...() methods be easy to implement? They would sure be handy to have from a user interface perspective. (Mike M) (Done for the ones in this completed section, but still an ongoing effort). # Add Resouce``BLOB to Resource (Andrew). # Add Keywords to Data``Container through servlet (Andrew) (This should work now, but not really tested) # Query for Data``Container by Keyword (Andrew) (Done, but I am going to refactor to implement one query method like I did in Data``Producer``DAO). # Query for Data``Container by Lat/Lon and by time (Andrew) (Done, but I am going to refactor to implement one query method like I did in Data``Producer``DAO). # Object/XML Builder done for new architecture. # Metadata``Access``Servlet updated for new architecture # Fixed bug in Metadata``Factory to return primitive classes when primitive text is passed in. # Fixed DTS job to copy from old database to new on Fog (1/11/2006) # Change SSDS so it can handle new Summary``Packet from SIAM. No change was necessary as Tom is exporting Summary``Packets as Sensor``Data``Packets # Figure out if we need to submit any text for SSDS Project for Annual Report (due 2/1) # Get Andrew's Metadata``Access``Servlet documentation up on servlet and/or ssds web application # Write up new architecture and migration path for new machine and architecture deployment. # Simplify operation tasks (startup, shutdown, health, etc.) # Document requirements for data access for MSE and get those and schedule to Keith ## Talk to Charlie, Jim Barry, Bill Ussler, Rendy, Chris Lovera about how they process data from the instruments currently. ## Create list of UI goals for this (and beyond) and figure out taks/skills for that. # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## On plots, link sensor manufacturer and model and put on plot itself (if possible) ## If min/max value exist for record variables, clip generated plots to those values. Also can use the new displayMin/displayMax variables where data gaps are) # Look into Ken's suggestion about ID's for deployment. From his email {noformat} All, When looking at the raw data in SSDS is can be very difficult to figure out what is what mooring and will get even more so as time goes by. This email is an attempt to get discussions going on how to deal with eliminating the confusion. All instruments, including controller cans can (and have been) be swapped during the deployment. It would be nice to have an ID for each deployment i.e. 2004 M0, 2005 M0, 2005 M2.If this ID were set up in the can as the parent ID just before deployment (i.e. aboard ship on the way out), then the problem of trying to separate out test data from deployed data would be easier to manage. It should also help resolve the issue of having latitude and longitude data in pucks. Instead the mooring ID would have the location associated with it. Regards, Ken Heller {noformat} # Implement Query: Find all instruments currently deployed on a "deployment" # Implement Query: They want to be able to selectively view only Metdata``Packets (Record``Type) and find only the most recent one. # Implement Query: They want a way to know what is currently deployed (user interface). # Fix Java``Script bug in SIAM Raw Data Access Page: This is a bug that Andy Hamilton showed me. When you select start to end time option, the get data, then click on back, the Javascript diables some of the time selection fields, which then invalidate the query and if you try to get the data again, you get a Null``Pointer``Exception. # Follow up with ops group with meeting on ID's, device, sensors, etc. and clean up device DB in SSDS # Add way to lookup instruments by platform on the device table # Merge access method (SIAM``Raw``Data``Access and Get``Original``Data``Servlet) to have same options and same output, web page simply forms the URL. An email {noformat} Hi Kevin, I'm noticing that the GetOriginalDataServlet app seems to be adding an additional CR/LF between the ISUS samples (each sample has four data records) when I snarf it directly as opposed to displaying it via a browser. See the URL below to see what I'm describing. It won't be a problem as far as I'm concerned just something I noticed. http://www2.mbari.org/coletti/m0_isus.cgi Luke And another Ah Ha, look what happens when you set noHTMLHeader=0 (URL below)... http://ssds.shore.mbari.org:8080/access/GetOriginalDataServlet?platformID=1299&metadataID=0 &recordTypeID=1&isi=0&noHTMLHeader=0&deviceID=1268&startDateTime=20040603.223400&endDateTime=20040606.223400, Luke {noformat} # Implmement basic metadata/data QC internal to SSDS (watchdog) # Refactor Ruminate in new architecture # When information in XML is inconsistent (device ID) with that of packet, somebody should be notified. # If device comes in with new ID, it will be updated/persisted, should not' # Performance investigation of services (sort of ongoing process) # The XML``Metadata``Tracker does not look like it is working properly (especially on AUV files) in the ruminate process # Andrew metioned that the depths (Data``Container) are rounding. # Boolean not setting on incoming Metadata``Access``Servlet query (verify this is fixed in new architecture, then move to completed) # Connect up raw stream access to the SQL database stream storage # Add parameter that the user can ask for the last X number of records. # Can we setup the Record``Description so that it can define a regular expression that can parse the whole record into variables? # Can we have a service that will automatically take NEMA strings from GPS's and convert them to a more useful format? # Implement Time``Indexed``Net``CDF``Access class # Crawl all data stream files and republish data to SQL topic to populate DB with device stream data. # Get NetCDF Watchdog going to keep NetCDF files created for those Data``Containers that ask for them. # Get XML Defined for the Nortek binary case (variable binary length) # Data servlet variables out of order and no flag for units # Fix Bob's Code so it queries by deployment name, not device name then change 1305 and 1414 device names back to OASIS Bouy.Then go back and clean up device list per Paul C. (Remove some of the M1/M2 designations). Remove devices 1307 and 1308 from database. # Re-run all loads for AUVCTD data onto the production machine to make sure it is all there(?) # When SSDSPUB tries to read from dods.mbari.org, it times out. h3. Future Tasks (Shelved) # Give Nancy Barr a base URL and she can have one of her crawlers check the links # Look at porting code to J2SE 5 (Generics, Iterators). # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Develop method for handling app deployment on heckel/jeckel # Can we use Bob Herlien to do some development work? # Get Mike M and Dave from (CenCOOS) and talk through SSDS for CenCOOS and getting them AUV data from MBARI. # Apply AJAX to web pages where performance problems exist ([http://code.jalenack.com/periodic/ Example]) # Fix Metadata links on CIMT pages to make them dynamic (or replace with different view, not XML) # Pull the MBARI ADCP page off the CIMT site # Look at [http://www.sensescape.com/scivis/index.html Visualization in Marine Science] # Look into [http://www.openmi.org OpenMI] # Turn on wrapper generator unit tests in continuous integration and process/display results # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## Make any direction plot (wind, heading, etc.) plot as points, not lines ## Put nominal lattitude and longitude in plot titles ## Have capability to turn on/off autoscale on plots and specify range # Get UI Design meetings going with the ops group and get portal prototype going # Ken Heller came by to ask to turn on more batch created plots for CIMT # Put link to ACE jar in web application # Ken also had another request to be able to see all variables plotted as thumbnails on one page, be able to select various plots and then have them show as full size together on another page (a merging of sorts for data). # Add pages for metadata editing (MetadataEditingNote) # SIAM``Raw``Data``Access Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement Query: Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location). This is specifically to support uploading the JARS for SIAM and linking them to Devices and Data``Producers(Deployments). # Implement Query: They would like to be able to store the driver class name that is linked to a Device so that they can have the user simply choose a device and it will start with that driver name. # Instead of using comand line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into having SSDS create "README" type files in the same location as certain Data``Containers (This could be a perl script bot) # Hold Brown Bag on architecture and uses. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional Standard``Variables, Standard``Units, Standard``Keywords, Standard``Domain, Standard``Reference``Scale, Device``Type, Resource``Type, Data``Producer``Group, Data``Container``Group and then notify the user of that change so they can change their source. # Remove Deployment info from PUCK XML (someday, probably for MSE). # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. # Write tests for Resource``BLOB->Object``Builder for byte array and verify that it is working correctly. # Send API's for SSDS ingest and Data``Stream Access to Data``Stream LOOKING group # Look at Mike Godin's Schema changes for AOSN # Can I embed the business logic documentation as Java``Doc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # See if I can use NT logins with Merlia driver in SSDS # Have the capability to turn off emails when resubmit is done. # Implmement generic way of sending data/XML ## Create application that reads in NetCDF file and generates XML to match our schema # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Cut down on JAR file bloat (happening in new arch) # Crawl all the old OASIS files and load the metadata into SSDS. # Meet with MMI to talk about pulling our SV and SU from ontologies. # XML with schemas declared not validating # Add mechanism that users can drop files of known type with all metadata in known format into a directory and SSDS will automatically register and store it. # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # Implement ingest socket -> JMS topic # Add QC flag to SSDS packets and match in database (SSDS_Data). # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # In Packet``Output``Manager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and Packet``Outputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add end of line terminator as separator in parsing packet records (not files) # Can I create something that will emulate reading from a serial port but is back by the SSDS data service? This would be done so that native applications that are used to reading from serial ports, can still do so. To be honest, this would be really cool, but seems fragile and unlikely. This started with the Nortek, so look at Nortek.no for what they called "Online applications" # Get LOBO data into SSDS # Get raw data porting from SSDS to LAS server working for Mooring data (Leverage Mike G's stuff). # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" data by storing comments with the data and returning that to user's upon request (think NetCDFs history) # Edit bad packets that are serialized to disk # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data) # Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Setup service that will calculate salinity automatically. # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Query: "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Create Process to create useable data file from inductive CTD string # Look at integrating Earth Google using services. # HOOVES Improvements: ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface # Look at NPS AUV Mission Planner as a way to interact with AUV data.Don Brutzman and Peter Flynn came to give a demo on this and it seems to be a possibility for a tool that can integrate and visualize modeling and collected data (as well as vehicle tracks). Just check Don's web address to find information ]]> 1179742 8093826 {code} # It's recommended that the final XML is to be checked into the puckxml module in MBARI's CVS on moonjelly. # Submit the Metadata to SSDS using the SSDSLoads application (available at [http://new-ssds.mbari.org/ssds-docs/client/]. The \-h option provides a usage note): {code} java -jar ssdsLoads-new-ssds.jar -d 1696.xml {code} ## If a mistake is made in the metadata (e.g. forgetting the platform Deployment) it may be easier to undo the submission and start over - do a deep delete on the parent deployment if this is the case. ## Minor attribute fixes may be done by editing the database once the deployments have been loaded h2. Establish data publishing application # This step assumes that there will be some application reading data from the deployed instrument. Perhaps it is a legacy application which reads the data to load into a custom data storage or visualization system. For this application to publish data to SSDS it must have visibility of each record the instrument produces as that record needs to be "packaged" into a SensorDataPacket and "published" to SSDS. # An example Java application that will package and publish a record is below: # {code} import java.io.IOException; import moos.ssds.jms.PublisherComponent; import moos.ssds.transmogrify.SSDSDevicePacket; /** *

* Publish instrument data records to the SSDS database. The client must provide * SSDS device ID, timeStamp, sequence number, and payload. *

*
* * @author : $Author: mccann $ * @version : $Revision: 1.17.2.7 $ *
*

* The * Monterey Bay Aquarium Research Institute (MBARI) provides this * documentation and code "as is", with no warranty, express * or implied, of its quality or consistency. It is provided without * support and without obligation on the part of MBARI to assist in its * use, correction, modification, or enhancement. This information * should not be published or distributed to third parties without * specific written permission from MBARI. *

*
* Copyright 2008 MBARI.
* MBARI Proprietary Information. All rights reserved.
*
*
*/ /** * @author mccann * */ public class SsdsPublisher { /** * Publish data record from an instrument to SSDS * * @param deviceID * is the SSDS Device ID for the instrument that produces the data in payload * @param epochMilliseconds * time of payload sample in milliseconds since 1/1/1970 0000 GMT * @param sequenceNumber * an incrementing number for each packet * @param payload * data from instrument */ public static void publish(long deviceID, long epochMilliseconds, long sequenceNumber, String payload) { // Create a publisher component PublisherComponent pc = new PublisherComponent(); // Create a new SensorDataPacket SSDSDevicePacket packetToSend = new SSDSDevicePacket(deviceID, payload .getBytes().length); // Set the packet type to data (0 = Metadata, 1 = Data, 2 = Message) packetToSend.setPacketType(1); // Assign the time packetToSend.setSystemTime(epochMilliseconds); // Set the parentID to 0 for a parentless deployment packetToSend.setPlatformID(0); // Set the metadataref number to 0, if the data format changes and we can // refer to a different metadata packet then this number will change. packetToSend.setMetadataRef(0); // Set the sequence number packetToSend.setSequenceNo(sequenceNumber); // Set the payload packetToSend.setDataBuffer(payload.getBytes()); // Set the record type to 1 as this is the most common situation packetToSend.setRecordType(1); try { pc.publishBytes(SSDSDevicePacket .convertToPublishableByteArray(packetToSend)); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } /** * Test of SsdsPublisher.publish() * * @param args * No arguments are taken. Test values hard coded. */ public static void main(String[] args) { /* * Example packet for EITS instrument */ long eitsDeviceID = 1696; // 1696 is actual SSDS deviceID for EITS long sampleTime = 1228431283209L; // Sample time: 2008-12-04 22:54:43 long sampleSequenceNumber = 1; // May set it to ID from DB insert // Payload values must match the RecordVariables in SSDS String samplePayload = "12.2, 4.3, 768.2, 2.3, 1.2, 0.2"; // Temp, Cond, // Pres, U, // V, W SsdsPublisher.publish(eitsDeviceID, sampleTime, sampleSequenceNumber, samplePayload); } } {code}]]> 8061063 1212522 SSDS_Metadata to Solstice->SSDS_Metadata to match the one used for SSDSDevPC and run. ## Startup new-ssds jboss. ## Run all MSE Urls. ## Have Mike run Stanard``Variable updates. ## Rebuild code and deploy with messaging turned back on. ## Verify all data of interest for MSE is being plotted ## Move MTM3 web apps to same form as MSE. ## New Device creation/editing page ## New Person creation/editing page ## Point old jsp's to new jsp's ## Point old Get``Original``Data``Servlet (ssds.shore and ssdpsub) to new Get``Original``Data``Servlet ## Migrate HOOVES. ## Shut off ruminate on predator. ## Shut off cimt on predator. ## Undeploy all SSDS stuff except for access.war that has forwards in it. ## Verify what Luis is using for Salinity is just Get``Original``Data``Servlet. ## Leave for a bit, but remove SSDS from solstice. ## Remove SSDS from Fog. ## Clean out do fresh installation on prey (retask). ## Turn off replication of data from tornado to ssdspub ## Remove SSDS data database from SSDSPub and all that should remain is a few .war files pointing to new-ssds. ## Ensure M0 PC02 Plots working (M0 Turnaround) # Break up deployments in metadata for Mooring into pre-deployment and deployment (M0 is the real issue) (no progress, re-iterated by Francisco) # In build, fix so that the jsf-lib directory is removed and install jstl.jar & standard.jar # What is the official MSE name? (MTM Meeting next week). # Find NetCDF Files for CIMT data and send URLs to Brian Fulfrost # Web page access to data from final ADCP netcdf file (Mike M completed) # Find location and put XML schemas (and associated DTD's out on the public web). Also include that as part of the build process (http://new-ssds.mbari.org/ssds-docs/xml/schema/SSDS_Metadata.xsd) # Add uri attribute to Standard``Variable (Valid URI) # Alternate primary key on SV (name, uri) # Can we implement a method on Process``Run``Access that can do cascade deletes? (done in new architecture on Data``Producer``DAO and is called "deepDelete") # "I'm calling Process``Run``Access.update() with additional inputs and they are not getting added. (new architecture does this) # Get the graphs on the CIMT web site working with device 1480. # The Time/Date box in the SIAM``Raw``Data``Access``Page states it is PST, but that is no longer true since we moved to having the Jboss instance running in GMT. # Implement findWithinDateRange() (Mike M) method is called findByDateRangeAndName and can be invoked with null name to just search by date. # Implement findByDataProducerGroupName() [remove findByLikeDataProducerGroupName() & other ..like.. methods] (Mike M) # Return count methods, e.g. countOfFindByDataProducerType() (Mike M) (ongoing, but done for the ones listed here) # findByLikeNameWithinTimeAndWithinGeospatialCube() [remove the ..like.. & use exactMatch boolean] (Mike M) (Done and method is called findByNameAndTimeAndGeospatialCube). # Would these countOf...() methods be easy to implement? They would sure be handy to have from a user interface perspective. (Mike M) (Done for the ones in this completed section, but still an ongoing effort). # Add Resouce``BLOB to Resource (Andrew). # Add Keywords to Data``Container through servlet (Andrew) (This should work now, but not really tested) # Query for Data``Container by Keyword (Andrew) (Done, but I am going to refactor to implement one query method like I did in Data``Producer``DAO). # Query for Data``Container by Lat/Lon and by time (Andrew) (Done, but I am going to refactor to implement one query method like I did in Data``Producer``DAO). # Object/XML Builder done for new architecture. # Metadata``Access``Servlet updated for new architecture # Fixed bug in Metadata``Factory to return primitive classes when primitive text is passed in. # Fixed DTS job to copy from old database to new on Fog (1/11/2006) # Change SSDS so it can handle new Summary``Packet from SIAM. No change was necessary as Tom is exporting Summary``Packets as Sensor``Data``Packets # Figure out if we need to submit any text for SSDS Project for Annual Report (due 2/1) # Get Andrew's Metadata``Access``Servlet documentation up on servlet and/or ssds web application # Write up new architecture and migration path for new machine and architecture deployment. # Simplify operation tasks (startup, shutdown, health, etc.) # Document requirements for data access for MSE and get those and schedule to Keith ## Talk to Charlie, Jim Barry, Bill Ussler, Rendy, Chris Lovera about how they process data from the instruments currently. ## Create list of UI goals for this (and beyond) and figure out taks/skills for that. # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## On plots, link sensor manufacturer and model and put on plot itself (if possible) ## If min/max value exist for record variables, clip generated plots to those values. Also can use the new displayMin/displayMax variables where data gaps are) # Look into Ken's suggestion about ID's for deployment. From his email {noformat} All, When looking at the raw data in SSDS is can be very difficult to figure out what is what mooring and will get even more so as time goes by. This email is an attempt to get discussions going on how to deal with eliminating the confusion. All instruments, including controller cans can (and have been) be swapped during the deployment. It would be nice to have an ID for each deployment i.e. 2004 M0, 2005 M0, 2005 M2.If this ID were set up in the can as the parent ID just before deployment (i.e. aboard ship on the way out), then the problem of trying to separate out test data from deployed data would be easier to manage. It should also help resolve the issue of having latitude and longitude data in pucks. Instead the mooring ID would have the location associated with it. Regards, Ken Heller {noformat} # Implement Query: Find all instruments currently deployed on a "deployment" # Implement Query: They want to be able to selectively view only Metdata``Packets (Record``Type) and find only the most recent one. # Implement Query: They want a way to know what is currently deployed (user interface). # Fix Java``Script bug in SIAM Raw Data Access Page: This is a bug that Andy Hamilton showed me. When you select start to end time option, the get data, then click on back, the Javascript diables some of the time selection fields, which then invalidate the query and if you try to get the data again, you get a Null``Pointer``Exception. # Follow up with ops group with meeting on ID's, device, sensors, etc. and clean up device DB in SSDS # Add way to lookup instruments by platform on the device table # Merge access method (SIAM``Raw``Data``Access and Get``Original``Data``Servlet) to have same options and same output, web page simply forms the URL. An email {noformat} Hi Kevin, I'm noticing that the GetOriginalDataServlet app seems to be adding an additional CR/LF between the ISUS samples (each sample has four data records) when I snarf it directly as opposed to displaying it via a browser. See the URL below to see what I'm describing. It won't be a problem as far as I'm concerned just something I noticed. http://www2.mbari.org/coletti/m0_isus.cgi Luke And another Ah Ha, look what happens when you set noHTMLHeader=0 (URL below)... http://ssds.shore.mbari.org:8080/access/GetOriginalDataServlet?platformID=1299&metadataID=0 &recordTypeID=1&isi=0&noHTMLHeader=0&deviceID=1268&startDateTime=20040603.223400&endDateTime=20040606.223400, Luke {noformat} # Implmement basic metadata/data QC internal to SSDS (watchdog) # Refactor Ruminate in new architecture # When information in XML is inconsistent (device ID) with that of packet, somebody should be notified. # If device comes in with new ID, it will be updated/persisted, should not' # Performance investigation of services (sort of ongoing process) # The XML``Metadata``Tracker does not look like it is working properly (especially on AUV files) in the ruminate process # Andrew metioned that the depths (Data``Container) are rounding. # Boolean not setting on incoming Metadata``Access``Servlet query (verify this is fixed in new architecture, then move to completed) # Connect up raw stream access to the SQL database stream storage # Add parameter that the user can ask for the last X number of records. # Can we setup the Record``Description so that it can define a regular expression that can parse the whole record into variables? # Can we have a service that will automatically take NEMA strings from GPS's and convert them to a more useful format? # Implement Time``Indexed``Net``CDF``Access class # Crawl all data stream files and republish data to SQL topic to populate DB with device stream data. # Get NetCDF Watchdog going to keep NetCDF files created for those Data``Containers that ask for them. # Get XML Defined for the Nortek binary case (variable binary length) # Data servlet variables out of order and no flag for units # Fix Bob's Code so it queries by deployment name, not device name then change 1305 and 1414 device names back to OASIS Bouy.Then go back and clean up device list per Paul C. (Remove some of the M1/M2 designations). Remove devices 1307 and 1308 from database. # Re-run all loads for AUVCTD data onto the production machine to make sure it is all there(?) # When SSDSPUB tries to read from dods.mbari.org, it times out. h3. Future Tasks (Shelved) # Give Nancy Barr a base URL and she can have one of her crawlers check the links # Look at porting code to J2SE 5 (Generics, Iterators). # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Develop method for handling app deployment on heckel/jeckel # Can we use Bob Herlien to do some development work? # Get Mike M and Dave from (CenCOOS) and talk through SSDS for CenCOOS and getting them AUV data from MBARI. # Apply AJAX to web pages where performance problems exist ([http://code.jalenack.com/periodic/ Example]) # Fix Metadata links on CIMT pages to make them dynamic (or replace with different view, not XML) # Pull the MBARI ADCP page off the CIMT site # Look at [http://www.sensescape.com/scivis/index.html Visualization in Marine Science] # Look into [http://www.openmi.org OpenMI] # Turn on wrapper generator unit tests in continuous integration and process/display results # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## Make any direction plot (wind, heading, etc.) plot as points, not lines ## Put nominal lattitude and longitude in plot titles ## Have capability to turn on/off autoscale on plots and specify range # Get UI Design meetings going with the ops group and get portal prototype going # Ken Heller came by to ask to turn on more batch created plots for CIMT # Put link to ACE jar in web application # Ken also had another request to be able to see all variables plotted as thumbnails on one page, be able to select various plots and then have them show as full size together on another page (a merging of sorts for data). # Add pages for metadata editing (MetadataEditingNote) # SIAM``Raw``Data``Access Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement Query: Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location). This is specifically to support uploading the JARS for SIAM and linking them to Devices and Data``Producers(Deployments). # Implement Query: They would like to be able to store the driver class name that is linked to a Device so that they can have the user simply choose a device and it will start with that driver name. # Instead of using comand line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into having SSDS create "README" type files in the same location as certain Data``Containers (This could be a perl script bot) # Hold Brown Bag on architecture and uses. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional Standard``Variables, Standard``Units, Standard``Keywords, Standard``Domain, Standard``Reference``Scale, Device``Type, Resource``Type, Data``Producer``Group, Data``Container``Group and then notify the user of that change so they can change their source. # Remove Deployment info from PUCK XML (someday, probably for MSE). # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. # Write tests for Resource``BLOB->Object``Builder for byte array and verify that it is working correctly. # Send API's for SSDS ingest and Data``Stream Access to Data``Stream LOOKING group # Look at Mike Godin's Schema changes for AOSN # Can I embed the business logic documentation as Java``Doc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # See if I can use NT logins with Merlia driver in SSDS # Have the capability to turn off emails when resubmit is done. # Implmement generic way of sending data/XML ## Create application that reads in NetCDF file and generates XML to match our schema # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Cut down on JAR file bloat (happening in new arch) # Crawl all the old OASIS files and load the metadata into SSDS. # Meet with MMI to talk about pulling our SV and SU from ontologies. # XML with schemas declared not validating # Add mechanism that users can drop files of known type with all metadata in known format into a directory and SSDS will automatically register and store it. # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # Implement ingest socket -> JMS topic # Add QC flag to SSDS packets and match in database (SSDS_Data). # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # In Packet``Output``Manager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and Packet``Outputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add end of line terminator as separator in parsing packet records (not files) # Can I create something that will emulate reading from a serial port but is back by the SSDS data service? This would be done so that native applications that are used to reading from serial ports, can still do so. To be honest, this would be really cool, but seems fragile and unlikely. This started with the Nortek, so look at Nortek.no for what they called "Online applications" # Get LOBO data into SSDS # Get raw data porting from SSDS to LAS server working for Mooring data (Leverage Mike G's stuff). # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" data by storing comments with the data and returning that to user's upon request (think NetCDFs history) # Edit bad packets that are serialized to disk # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data) # Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Setup service that will calculate salinity automatically. # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Query: "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Create Process to create useable data file from inductive CTD string # Look at integrating Earth Google using services. # HOOVES Improvements: ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface # Look at NPS AUV Mission Planner as a way to interact with AUV data.Don Brutzman and Peter Flynn came to give a demo on this and it seems to be a possibility for a tool that can integrate and visualize modeling and collected data (as well as vehicle tracks). Just check Don's web address to find information ]]> 1179755 1212521 SSDS_Metadata to Solstice->SSDS_Metadata to match the one used for SSDSDevPC and run. ## Startup new-ssds jboss. ## Run all MSE Urls. ## Have Mike run Stanard``Variable updates. ## Rebuild code and deploy with messaging turned back on. ## Verify all data of interest for MSE is being plotted ## Move MTM3 web apps to same form as MSE. ## New Device creation/editing page ## New Person creation/editing page ## Point old jsp's to new jsp's ## Point old Get``Original``Data``Servlet (ssds.shore and ssdpsub) to new Get``Original``Data``Servlet ## Migrate HOOVES. ## Shut off ruminate on predator. ## Shut off cimt on predator. ## Undeploy all SSDS stuff except for access.war that has forwards in it. ## Verify what Luis is using for Salinity is just Get``Original``Data``Servlet. ## Leave for a bit, but remove SSDS from solstice. ## Remove SSDS from Fog. ## Clean out do fresh installation on prey (retask). ## Turn off replication of data from tornado to ssdspub ## Remove SSDS data database from SSDSPub and all that should remain is a few .war files pointing to new-ssds. ## Ensure M0 PC02 Plots working (M0 Turnaround) # Break up deployments in metadata for Mooring into pre-deployment and deployment (M0 is the real issue) (no progress, re-iterated by Francisco) # In build, fix so that the jsf-lib directory is removed and install jstl.jar & standard.jar # What is the official MSE name? (MTM Meeting next week). # Find NetCDF Files for CIMT data and send URLs to Brian Fulfrost # Web page access to data from final ADCP netcdf file (Mike M completed) # Find location and put XML schemas (and associated DTD's out on the public web). Also include that as part of the build process (http://new-ssds.mbari.org/ssds-docs/xml/schema/SSDS_Metadata.xsd) # Add uri attribute to Standard``Variable (Valid URI) # Alternate primary key on SV (name, uri) # Can we implement a method on Process``Run``Access that can do cascade deletes? (done in new architecture on Data``Producer``DAO and is called "deepDelete") # "I'm calling Process``Run``Access.update() with additional inputs and they are not getting added. (new architecture does this) # Get the graphs on the CIMT web site working with device 1480. # The Time/Date box in the SIAM``Raw``Data``Access``Page states it is PST, but that is no longer true since we moved to having the Jboss instance running in GMT. # Implement findWithinDateRange() (Mike M) method is called findByDateRangeAndName and can be invoked with null name to just search by date. # Implement findByDataProducerGroupName() [remove findByLikeDataProducerGroupName() & other ..like.. methods] (Mike M) # Return count methods, e.g. countOfFindByDataProducerType() (Mike M) (ongoing, but done for the ones listed here) # findByLikeNameWithinTimeAndWithinGeospatialCube() [remove the ..like.. & use exactMatch boolean] (Mike M) (Done and method is called findByNameAndTimeAndGeospatialCube). # Would these countOf...() methods be easy to implement? They would sure be handy to have from a user interface perspective. (Mike M) (Done for the ones in this completed section, but still an ongoing effort). # Add Resouce``BLOB to Resource (Andrew). # Add Keywords to Data``Container through servlet (Andrew) (This should work now, but not really tested) # Query for Data``Container by Keyword (Andrew) (Done, but I am going to refactor to implement one query method like I did in Data``Producer``DAO). # Query for Data``Container by Lat/Lon and by time (Andrew) (Done, but I am going to refactor to implement one query method like I did in Data``Producer``DAO). # Object/XML Builder done for new architecture. # Metadata``Access``Servlet updated for new architecture # Fixed bug in Metadata``Factory to return primitive classes when primitive text is passed in. # Fixed DTS job to copy from old database to new on Fog (1/11/2006) # Change SSDS so it can handle new Summary``Packet from SIAM. No change was necessary as Tom is exporting Summary``Packets as Sensor``Data``Packets # Figure out if we need to submit any text for SSDS Project for Annual Report (due 2/1) # Get Andrew's Metadata``Access``Servlet documentation up on servlet and/or ssds web application # Write up new architecture and migration path for new machine and architecture deployment. # Simplify operation tasks (startup, shutdown, health, etc.) # Document requirements for data access for MSE and get those and schedule to Keith ## Talk to Charlie, Jim Barry, Bill Ussler, Rendy, Chris Lovera about how they process data from the instruments currently. ## Create list of UI goals for this (and beyond) and figure out taks/skills for that. # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## On plots, link sensor manufacturer and model and put on plot itself (if possible) ## If min/max value exist for record variables, clip generated plots to those values. Also can use the new displayMin/displayMax variables where data gaps are) # Look into Ken's suggestion about ID's for deployment. From his email {noformat} All, When looking at the raw data in SSDS is can be very difficult to figure out what is what mooring and will get even more so as time goes by. This email is an attempt to get discussions going on how to deal with eliminating the confusion. All instruments, including controller cans can (and have been) be swapped during the deployment. It would be nice to have an ID for each deployment i.e. 2004 M0, 2005 M0, 2005 M2.If this ID were set up in the can as the parent ID just before deployment (i.e. aboard ship on the way out), then the problem of trying to separate out test data from deployed data would be easier to manage. It should also help resolve the issue of having latitude and longitude data in pucks. Instead the mooring ID would have the location associated with it. Regards, Ken Heller {noformat} # Implement Query: Find all instruments currently deployed on a "deployment" # Implement Query: They want to be able to selectively view only Metdata``Packets (Record``Type) and find only the most recent one. # Implement Query: They want a way to know what is currently deployed (user interface). # Fix Java``Script bug in SIAM Raw Data Access Page: This is a bug that Andy Hamilton showed me. When you select start to end time option, the get data, then click on back, the Javascript diables some of the time selection fields, which then invalidate the query and if you try to get the data again, you get a Null``Pointer``Exception. # Follow up with ops group with meeting on ID's, device, sensors, etc. and clean up device DB in SSDS # Add way to lookup instruments by platform on the device table # Merge access method (SIAM``Raw``Data``Access and Get``Original``Data``Servlet) to have same options and same output, web page simply forms the URL. An email {noformat} Hi Kevin, I'm noticing that the GetOriginalDataServlet app seems to be adding an additional CR/LF between the ISUS samples (each sample has four data records) when I snarf it directly as opposed to displaying it via a browser. See the URL below to see what I'm describing. It won't be a problem as far as I'm concerned just something I noticed. http://www2.mbari.org/coletti/m0_isus.cgi Luke And another Ah Ha, look what happens when you set noHTMLHeader=0 (URL below)... http://ssds.shore.mbari.org:8080/access/GetOriginalDataServlet?platformID=1299&metadataID=0 &recordTypeID=1&isi=0&noHTMLHeader=0&deviceID=1268&startDateTime=20040603.223400&endDateTime=20040606.223400, Luke {noformat} # Implmement basic metadata/data QC internal to SSDS (watchdog) # Refactor Ruminate in new architecture # When information in XML is inconsistent (device ID) with that of packet, somebody should be notified. # If device comes in with new ID, it will be updated/persisted, should not' # Performance investigation of services (sort of ongoing process) # The XML``Metadata``Tracker does not look like it is working properly (especially on AUV files) in the ruminate process # Andrew metioned that the depths (Data``Container) are rounding. # Boolean not setting on incoming Metadata``Access``Servlet query (verify this is fixed in new architecture, then move to completed) # Connect up raw stream access to the SQL database stream storage # Add parameter that the user can ask for the last X number of records. # Can we setup the Record``Description so that it can define a regular expression that can parse the whole record into variables? # Can we have a service that will automatically take NEMA strings from GPS's and convert them to a more useful format? # Implement Time``Indexed``Net``CDF``Access class # Crawl all data stream files and republish data to SQL topic to populate DB with device stream data. # Get NetCDF Watchdog going to keep NetCDF files created for those Data``Containers that ask for them. # Get XML Defined for the Nortek binary case (variable binary length) # Data servlet variables out of order and no flag for units # Fix Bob's Code so it queries by deployment name, not device name then change 1305 and 1414 device names back to OASIS Bouy.Then go back and clean up device list per Paul C. (Remove some of the M1/M2 designations). Remove devices 1307 and 1308 from database. # Re-run all loads for AUVCTD data onto the production machine to make sure it is all there(?) # When SSDSPUB tries to read from dods.mbari.org, it times out. h3. Future Tasks (Shelved) # Give Nancy Barr a base URL and she can have one of her crawlers check the links # Look at porting code to J2SE 5 (Generics, Iterators). # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Develop method for handling app deployment on heckel/jeckel # Can we use Bob Herlien to do some development work? # Get Mike M and Dave from (CenCOOS) and talk through SSDS for CenCOOS and getting them AUV data from MBARI. # Apply AJAX to web pages where performance problems exist ([http://code.jalenack.com/periodic/ Example]) # Fix Metadata links on CIMT pages to make them dynamic (or replace with different view, not XML) # Pull the MBARI ADCP page off the CIMT site # Look at [http://www.sensescape.com/scivis/index.html Visualization in Marine Science] # Look into [http://www.openmi.org OpenMI] # Turn on wrapper generator unit tests in continuous integration and process/display results # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## Make any direction plot (wind, heading, etc.) plot as points, not lines ## Put nominal lattitude and longitude in plot titles ## Have capability to turn on/off autoscale on plots and specify range # Get UI Design meetings going with the ops group and get portal prototype going # Ken Heller came by to ask to turn on more batch created plots for CIMT # Put link to ACE jar in web application # Ken also had another request to be able to see all variables plotted as thumbnails on one page, be able to select various plots and then have them show as full size together on another page (a merging of sorts for data). # Add pages for metadata editing (MetadataEditingNote) # SIAM``Raw``Data``Access Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement Query: Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location). This is specifically to support uploading the JARS for SIAM and linking them to Devices and Data``Producers(Deployments). # Implement Query: They would like to be able to store the driver class name that is linked to a Device so that they can have the user simply choose a device and it will start with that driver name. # Instead of using comand line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into having SSDS create "README" type files in the same location as certain Data``Containers (This could be a perl script bot) # Hold Brown Bag on architecture and uses. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional Standard``Variables, Standard``Units, Standard``Keywords, Standard``Domain, Standard``Reference``Scale, Device``Type, Resource``Type, Data``Producer``Group, Data``Container``Group and then notify the user of that change so they can change their source. # Remove Deployment info from PUCK XML (someday, probably for MSE). # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. # Write tests for Resource``BLOB->Object``Builder for byte array and verify that it is working correctly. # Send API's for SSDS ingest and Data``Stream Access to Data``Stream LOOKING group # Look at Mike Godin's Schema changes for AOSN # Can I embed the business logic documentation as Java``Doc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # See if I can use NT logins with Merlia driver in SSDS # Have the capability to turn off emails when resubmit is done. # Implmement generic way of sending data/XML ## Create application that reads in NetCDF file and generates XML to match our schema # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Cut down on JAR file bloat (happening in new arch) # Crawl all the old OASIS files and load the metadata into SSDS. # Meet with MMI to talk about pulling our SV and SU from ontologies. # XML with schemas declared not validating # Add mechanism that users can drop files of known type with all metadata in known format into a directory and SSDS will automatically register and store it. # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # Implement ingest socket -> JMS topic # Add QC flag to SSDS packets and match in database (SSDS_Data). # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # In Packet``Output``Manager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and Packet``Outputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add end of line terminator as separator in parsing packet records (not files) # Can I create something that will emulate reading from a serial port but is back by the SSDS data service? This would be done so that native applications that are used to reading from serial ports, can still do so. To be honest, this would be really cool, but seems fragile and unlikely. This started with the Nortek, so look at Nortek.no for what they called "Online applications" # Get LOBO data into SSDS # Get raw data porting from SSDS to LAS server working for Mooring data (Leverage Mike G's stuff). # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" data by storing comments with the data and returning that to user's upon request (think NetCDFs history) # Edit bad packets that are serialized to disk # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data) # Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Setup service that will calculate salinity automatically. # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Query: "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Create Process to create useable data file from inductive CTD string # Look at integrating Earth Google using services. # HOOVES Improvements: ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface # Look at NPS AUV Mission Planner as a way to interact with AUV data.Don Brutzman and Peter Flynn came to give a demo on this and it seems to be a possibility for a tool that can integrate and visualize modeling and collected data (as well as vehicle tracks). Just check Don's web address to find information ]]> 1179754 1212530 Fog-SSDS_Metadata h3. UpdateBot # Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) # Have updateBot crawl all resources and update contentLength if not specified. # Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. # Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). h3. Graphing Application # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). h3. Transmogrify/Ingest/SQLIngest # Build non-JMS mechanism for users to send data/metadata to SSDS. # Put true exception handling in SQLIngest to throw messages when the DB can't be reached. # Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). h3. Core (EAR) # Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds # Build services to read data from DataContainers that are files through the query interface (not just from packets). # Verify (unit tests) that the RecordDescription level parse regular expression works # Finish implementing all DAOs # Make sure all methods have associated count method # Make sure all methods have boolean option for return full graph # Make sure all methods have capability to specify a sort by field # Verify returned DataContainer collections should be sorted by start date as default # Verify implemented query for DataContainer by DataContainerGroup # Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. # Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. # Write valid unit test for Object and XMLBuilders # Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. h3. Web applications # Verify PC02 plots are working after M0 turnaround # Develop admin application to edit all metadata objects and their relationships # Increase session timeout in Explorer # Add capability in Explorer to export deployment XML template from Web to help in XML authoring (or at least connect to most recent XML in CVS). ## Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Put UML diagram of data model on developer section of web app. # Implement more queries in Explorer ## Find all post products from deployment ## Find all resources of certain types (graphics, log files, calibration files, etc.) ## "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ## Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ## Given a deployment, can I walk up the parent tree to look for a parent with a certain role? # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Build web pages that allow user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Finish documenting data packet structure on web pages. # Web pages to help with automated workflows(?) # Desire in long term for common 'status reporting' system for instruments (something like an ops portal) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) h3. Clients # Migrate HOOVES to new architecture # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) h3. Other Project Overlaps h5. SNMP h5. ORION CI h5. CGSN h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Future Tasks (Shelved) # Give Nancy Barr a base URL and she can have one of her crawlers check the links # Look at porting code to J2SE 5 (Generics, Iterators). # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Develop method for handling app deployment on heckel/jeckel # Can we use Bob Herlien to do some development work? # Get Mike M and Dave from (CenCOOS) and talk through SSDS for CenCOOS and getting them AUV data from MBARI. # Apply AJAX to web pages where performance problems exist ([http://code.jalenack.com/periodic/ Example]) # Fix Metadata links on CIMT pages to make them dynamic (or replace with different view, not XML) # Pull the MBARI ADCP page off the CIMT site # Look at [http://www.sensescape.com/scivis/index.html Visualization in Marine Science] # Look into [http://www.openmi.org OpenMI] # Turn on wrapper generator unit tests in continuous integration and process/display results # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## Make any direction plot (wind, heading, etc.) plot as points, not lines ## Put nominal lattitude and longitude in plot titles ## Have capability to turn on/off autoscale on plots and specify range # Get UI Design meetings going with the ops group and get portal prototype going # Ken Heller came by to ask to turn on more batch created plots for CIMT # Put link to ACE jar in web application # Ken also had another request to be able to see all variables plotted as thumbnails on one page, be able to select various plots and then have them show as full size together on another page (a merging of sorts for data). # Add pages for metadata editing (MetadataEditingNote) # SIAM``Raw``Data``Access Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement Query: Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location). This is specifically to support uploading the JARS for SIAM and linking them to Devices and Data``Producers(Deployments). # Implement Query: They would like to be able to store the driver class name that is linked to a Device so that they can have the user simply choose a device and it will start with that driver name. # Instead of using comand line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into having SSDS create "README" type files in the same location as certain Data``Containers (This could be a perl script bot) # Hold Brown Bag on architecture and uses. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional Standard``Variables, Standard``Units, Standard``Keywords, Standard``Domain, Standard``Reference``Scale, Device``Type, Resource``Type, Data``Producer``Group, Data``Container``Group and then notify the user of that change so they can change their source. # Remove Deployment info from PUCK XML (someday, probably for MSE). # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. # Write tests for Resource``BLOB->Object``Builder for byte array and verify that it is working correctly. # Send API's for SSDS ingest and Data``Stream Access to Data``Stream LOOKING group # Look at Mike Godin's Schema changes for AOSN # Can I embed the business logic documentation as Java``Doc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # See if I can use NT logins with Merlia driver in SSDS # Have the capability to turn off emails when resubmit is done. # Implmement generic way of sending data/XML ## Create application that reads in NetCDF file and generates XML to match our schema # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Cut down on JAR file bloat (happening in new arch) # Crawl all the old OASIS files and load the metadata into SSDS. # Meet with MMI to talk about pulling our SV and SU from ontologies. # XML with schemas declared not validating # Add mechanism that users can drop files of known type with all metadata in known format into a directory and SSDS will automatically register and store it. # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # Implement ingest socket -> JMS topic # Add QC flag to SSDS packets and match in database (SSDS_Data). # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # In Packet``Output``Manager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and Packet``Outputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add end of line terminator as separator in parsing packet records (not files) # Can I create something that will emulate reading from a serial port but is back by the SSDS data service? This would be done so that native applications that are used to reading from serial ports, can still do so. To be honest, this would be really cool, but seems fragile and unlikely. This started with the Nortek, so look at Nortek.no for what they called "Online applications" # Get LOBO data into SSDS # Get raw data porting from SSDS to LAS server working for Mooring data (Leverage Mike G's stuff). # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" data by storing comments with the data and returning that to user's upon request (think NetCDFs history) # Edit bad packets that are serialized to disk # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data) # Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Setup service that will calculate salinity automatically. # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Query: "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Create Process to create useable data file from inductive CTD string # Look at integrating Earth Google using services. # HOOVES Improvements: ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface # Look at NPS AUV Mission Planner as a way to interact with AUV data.Don Brutzman and Peter Flynn came to give a demo on this and it seems to be a possibility for a tool that can integrate and visualize modeling and collected data (as well as vehicle tracks). Just check Don's web address to find information ]]> 1179763 1212527 Fog-SSDS_Metadata h3. UpdateBot # Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) # Have updateBot crawl all resources and update contentLength if not specified. # Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. # Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). h3. Transmogrify/Ingest/SQLIngest # Build non-JMS mechanism for users to send data/metadata to SSDS. # Put true exception handling in SQLIngest to throw messages when the DB can't be reached. # Change PacketSQLOutput/Input to work with any database (not just MS SQL) h3. Core (EAR) # Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds # Build services to read data from DataContainers that are files through the query interface (not just from packets). # Verify (unit tests) that the RecordDescription level parse regular expression works h3. Web applications # Verify PC02 plots are working after M0 turnaround # Develop admin application to edit all metadata objects and their relationships # Increase session timeout in Explorer # Add capability in Explorer to export deployment XML template from Web to help in XML authoring (or at least connect to most recent XML in CVS). ## Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Put UML diagram of data model on developer section of web app. # Implement more queries in Explorer ## Find all post products from deployment ## Find all resources of certain types (graphics, log files, calibration files, etc.) # Refactor navigation menus and scheme for SSDS site and post processed pages # Build web pages that allow user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Finish documenting data packet structure on web pages. h3. Clients # Migrate HOOVES to new architecture # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) h3. Other # QC Pages ## show instruments removed for service or currently reflecting invalid data (e.g., cover left on) (John has notes of which) ### show expected turn-on date if known (note: would have to keep this up) ### add "Planned availability xxxx" to missing data and displays on website so appears more individually organized to users then just a generic "data not available" ## show processed QC data wherever available -- this is very valuable feature ## fix graphs that don't size correctly on Mark's machine (probably cross-platform browser problem) ## look for other cross-platform issues ## fix multispectral page heading (says "Backscatter/chlorophyll") ## on chlorophyll page, turbidity isn't presented (get plots working) ## add ISUS data (get plots working) ## add parsing of Medusa data where possible ## make first page to load the MSE Sites page rather than the Domains page ### Benthic nodes in Sites pages have some instruments not actually on node (check) ## Update anchor as-deployed position (MSE anchor is 36.218321° N -122.904986° W) ## Scale both axes equally on GPS plots that are scaled to data. ## Make all metadata (.xml) links work ### Be nice if metadata page were improved (why not put files in body of page?) ## Improve graph names on plots (particularly on power pages, n.b., "ADCnn") ## Correct the formatting of titles on plots (add spaces for clarity) ## Add portal data items (particularly Globalstar download durations) to data pages when available ## Delete Diagnostics page ## Add GPS plot showing location since deployment ## Change "Platform Environment" to "Platform Status" throughout ## need to show plots of summary data on these pages as soon as reasonably possible ## fix CTD post-processed plots that aren't updating (point to 20060502 instead of 20060503) ## Sure would be nice to see the real wind direction (must be computed) ## Be nice if QC plots knew to pick pen up when data is skipped ## Multiple variables on one axis is desirable (e.g., for Ed M) ## Consider deleting Sensor pages # Other Actions ## Mark: make links from MSE and MOOS web sites to SSDS MSE pages (update those sites) ## Mark: Review words at http://ssdspub.mbari.org:8080/mse/index.jsp ## Mark: send updated anchor location to Graybeal for watch circle plot ## Tom/SIAM: add SIAM infeastructure information to SSDS as SIAM packets ### Globalstar ppp info ### clock offsets ## Kevin: confirm SIAM Packets is the right way to add the SIAM infrastructure data ### provide mechanism for ISI ID or other data source ID ## TBD (McCann et al?): Aquadopp post-processing very desirable ## Tom/John: provide serial info to Paul for MSE GPS, discuss MTM3 GPS serial/ISI number issues w/him ### Tom: fix appropriate node to reflect correct GPS ISI ID ### Kevin: move data appropriately ### John: confirm primary GPS plots update correctly, put primary plots at top again ## Kevin: turn off fail-mail for missing sensors ## John: send missing sensor list ## Kent, Tom: look at the download times (vs. the real budget) and figure out what to do ## John: write XML translator for summary data (check w/Kevin first) ## John: note who is doing which post-processing somewhere (keep track) # Other Notes ## SOON post-processing data should be publicly displayed if possible, encourage scientists to do this. ### Raw data by itself cannot be interpreted or ground-truthed by anyone but instrument builder. ### MSE policy is for all MSE data to be publicly available. ## Enthusiasm expressed for post-processed data ## Enthusiasm expressed for ability to drill down to deployed sensors and their data using Explorer ## The existing diagnostic info on sensor pages (e.g., on LWR) is just fine ## Desire in long term for common 'status reporting' system for instruments ## Need to clean up presentation of left navigation on main SSDS page ## Need to add rest of SSDS web page content to main SSDS pages ## Being able to search and sort instruments by current location/deployment is extremely valuable # Refactor/overhaul chart creation application ## Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) ## Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) ## Setup Device``QCPlot``Creator to create plots with multiple lines on one chart (and separate axes). # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # Implement Query "I want the Air Temperature (Standard``Variable.name) from the Data``Producer named "M1" from such and such to such and such a time." # Query: Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data``Producer of type Deployment? # Query: Given a deployment, can I walk up the parent tree to look for a parent with a certain role? # Figure out how we will integrate data files offloaded from instruments when they are recovered. # Site Based Pages for MSE (4 days) # Possibility of direct support of ESB work with NCSA/LOOKING h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Verify Done # Add boolean to all query methods for returning full object graphs. # Implement methods to returns counts on queries. # Add capability of specifying what to sort results by (property name) in the query methods. # Returned DataContainer collections should be sorted by start date as default # Query for DataContainer by DataContainerGroup # Make sure XML dates that are coming out of XML``Builder/Object``Builder cycle can be parsed by the XML``Date``Format. # Check Object Builder/XML builder to make sure it handles URI/URL/Uri``String/DODS``Url/X/Y/Z``offsets correctly. # Make sure object Schema/Object``Builder/XML``Builder only use one Data``Container/Data``Producer for output, input, consumer, tags. h3. Completed Tasks # Look at AUVCTD portal code and make sure it survives an SSDS restart # Finish architecture migration: ## Finish UpdateBot ### Source/Destination tracking for flagging NetCDF creation ### Email after processing ## Change unique key definition on Standard``Variable code ## Rebuild code, deploy on ssdsdevpc and let it build database on Fog. ## Fix/Run DTS to accomodate the new alternate primary key on Standard``Variable to be name and namespaceUri and copy data from SSDS on Solstice to SSDS_Metadata on Fog. ## Rebuild code base, deploy to new-ssds and let code build DB on Solstice SSDS_Metadata. (TURN OFF MESSAGING EMAIL TILL AFTER MIKE DOES STANDARD VARIABLE UPDATES). ## Rebuild code base, redeploy with Create DB off. ## Change DTS job from Fog->SSDS_Metadata to Solstice->SSDS_Metadata to match the one used for SSDSDevPC and run. ## Startup new-ssds jboss. ## Run all MSE Urls. ## Have Mike run Stanard``Variable updates. ## Rebuild code and deploy with messaging turned back on. ## Verify all data of interest for MSE is being plotted ## Move MTM3 web apps to same form as MSE. ## New Device creation/editing page ## New Person creation/editing page ## Point old jsp's to new jsp's ## Point old Get``Original``Data``Servlet (ssds.shore and ssdpsub) to new Get``Original``Data``Servlet ## Migrate HOOVES. ## Shut off ruminate on predator. ## Shut off cimt on predator. ## Undeploy all SSDS stuff except for access.war that has forwards in it. ## Verify what Luis is using for Salinity is just Get``Original``Data``Servlet. ## Leave for a bit, but remove SSDS from solstice. ## Remove SSDS from Fog. ## Clean out do fresh installation on prey (retask). ## Turn off replication of data from tornado to ssdspub ## Remove SSDS data database from SSDSPub and all that should remain is a few .war files pointing to new-ssds. ## Ensure M0 PC02 Plots working (M0 Turnaround) # Break up deployments in metadata for Mooring into pre-deployment and deployment (M0 is the real issue) (no progress, re-iterated by Francisco) # In build, fix so that the jsf-lib directory is removed and install jstl.jar & standard.jar # What is the official MSE name? (MTM Meeting next week). # Find NetCDF Files for CIMT data and send URLs to Brian Fulfrost # Web page access to data from final ADCP netcdf file (Mike M completed) # Find location and put XML schemas (and associated DTD's out on the public web). Also include that as part of the build process (http://new-ssds.mbari.org/ssds-docs/xml/schema/SSDS_Metadata.xsd) # Add uri attribute to Standard``Variable (Valid URI) # Alternate primary key on SV (name, uri) # Can we implement a method on Process``Run``Access that can do cascade deletes? (done in new architecture on Data``Producer``DAO and is called "deepDelete") # "I'm calling Process``Run``Access.update() with additional inputs and they are not getting added. (new architecture does this) # Get the graphs on the CIMT web site working with device 1480. # The Time/Date box in the SIAM``Raw``Data``Access``Page states it is PST, but that is no longer true since we moved to having the Jboss instance running in GMT. # Implement findWithinDateRange() (Mike M) method is called findByDateRangeAndName and can be invoked with null name to just search by date. # Implement findByDataProducerGroupName() [remove findByLikeDataProducerGroupName() & other ..like.. methods] (Mike M) # Return count methods, e.g. countOfFindByDataProducerType() (Mike M) (ongoing, but done for the ones listed here) # findByLikeNameWithinTimeAndWithinGeospatialCube() [remove the ..like.. & use exactMatch boolean] (Mike M) (Done and method is called findByNameAndTimeAndGeospatialCube). # Would these countOf...() methods be easy to implement? They would sure be handy to have from a user interface perspective. (Mike M) (Done for the ones in this completed section, but still an ongoing effort). # Add Resouce``BLOB to Resource (Andrew). # Add Keywords to Data``Container through servlet (Andrew) (This should work now, but not really tested) # Query for Data``Container by Keyword (Andrew) (Done, but I am going to refactor to implement one query method like I did in Data``Producer``DAO). # Query for Data``Container by Lat/Lon and by time (Andrew) (Done, but I am going to refactor to implement one query method like I did in Data``Producer``DAO). # Object/XML Builder done for new architecture. # Metadata``Access``Servlet updated for new architecture # Fixed bug in Metadata``Factory to return primitive classes when primitive text is passed in. # Fixed DTS job to copy from old database to new on Fog (1/11/2006) # Change SSDS so it can handle new Summary``Packet from SIAM. No change was necessary as Tom is exporting Summary``Packets as Sensor``Data``Packets # Figure out if we need to submit any text for SSDS Project for Annual Report (due 2/1) # Get Andrew's Metadata``Access``Servlet documentation up on servlet and/or ssds web application # Write up new architecture and migration path for new machine and architecture deployment. # Simplify operation tasks (startup, shutdown, health, etc.) # Document requirements for data access for MSE and get those and schedule to Keith ## Talk to Charlie, Jim Barry, Bill Ussler, Rendy, Chris Lovera about how they process data from the instruments currently. ## Create list of UI goals for this (and beyond) and figure out taks/skills for that. # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## On plots, link sensor manufacturer and model and put on plot itself (if possible) ## If min/max value exist for record variables, clip generated plots to those values. Also can use the new displayMin/displayMax variables where data gaps are) # Look into Ken's suggestion about ID's for deployment. From his email {noformat} All, When looking at the raw data in SSDS is can be very difficult to figure out what is what mooring and will get even more so as time goes by. This email is an attempt to get discussions going on how to deal with eliminating the confusion. All instruments, including controller cans can (and have been) be swapped during the deployment. It would be nice to have an ID for each deployment i.e. 2004 M0, 2005 M0, 2005 M2.If this ID were set up in the can as the parent ID just before deployment (i.e. aboard ship on the way out), then the problem of trying to separate out test data from deployed data would be easier to manage. It should also help resolve the issue of having latitude and longitude data in pucks. Instead the mooring ID would have the location associated with it. Regards, Ken Heller {noformat} # Implement Query: Find all instruments currently deployed on a "deployment" # Implement Query: They want to be able to selectively view only Metdata``Packets (Record``Type) and find only the most recent one. # Implement Query: They want a way to know what is currently deployed (user interface). # Fix Java``Script bug in SIAM Raw Data Access Page: This is a bug that Andy Hamilton showed me. When you select start to end time option, the get data, then click on back, the Javascript diables some of the time selection fields, which then invalidate the query and if you try to get the data again, you get a Null``Pointer``Exception. # Follow up with ops group with meeting on ID's, device, sensors, etc. and clean up device DB in SSDS # Add way to lookup instruments by platform on the device table # Merge access method (SIAM``Raw``Data``Access and Get``Original``Data``Servlet) to have same options and same output, web page simply forms the URL. An email {noformat} Hi Kevin, I'm noticing that the GetOriginalDataServlet app seems to be adding an additional CR/LF between the ISUS samples (each sample has four data records) when I snarf it directly as opposed to displaying it via a browser. See the URL below to see what I'm describing. It won't be a problem as far as I'm concerned just something I noticed. http://www2.mbari.org/coletti/m0_isus.cgi Luke And another Ah Ha, look what happens when you set noHTMLHeader=0 (URL below)... http://ssds.shore.mbari.org:8080/access/GetOriginalDataServlet?platformID=1299&metadataID=0 &recordTypeID=1&isi=0&noHTMLHeader=0&deviceID=1268&startDateTime=20040603.223400&endDateTime=20040606.223400, Luke {noformat} # Implmement basic metadata/data QC internal to SSDS (watchdog) # Refactor Ruminate in new architecture # When information in XML is inconsistent (device ID) with that of packet, somebody should be notified. # If device comes in with new ID, it will be updated/persisted, should not' # Performance investigation of services (sort of ongoing process) # The XML``Metadata``Tracker does not look like it is working properly (especially on AUV files) in the ruminate process # Andrew metioned that the depths (Data``Container) are rounding. # Boolean not setting on incoming Metadata``Access``Servlet query (verify this is fixed in new architecture, then move to completed) # Connect up raw stream access to the SQL database stream storage # Add parameter that the user can ask for the last X number of records. # Can we setup the Record``Description so that it can define a regular expression that can parse the whole record into variables? # Can we have a service that will automatically take NEMA strings from GPS's and convert them to a more useful format? # Implement Time``Indexed``Net``CDF``Access class # Crawl all data stream files and republish data to SQL topic to populate DB with device stream data. # Get NetCDF Watchdog going to keep NetCDF files created for those Data``Containers that ask for them. # Get XML Defined for the Nortek binary case (variable binary length) # Data servlet variables out of order and no flag for units # Fix Bob's Code so it queries by deployment name, not device name then change 1305 and 1414 device names back to OASIS Bouy.Then go back and clean up device list per Paul C. (Remove some of the M1/M2 designations). Remove devices 1307 and 1308 from database. # Re-run all loads for AUVCTD data onto the production machine to make sure it is all there(?) # When SSDSPUB tries to read from dods.mbari.org, it times out. # Verify what Luis is using for Salinity is just Get``Original``Data``Servlet. h3. Future Tasks (Shelved) # Give Nancy Barr a base URL and she can have one of her crawlers check the links # Look at porting code to J2SE 5 (Generics, Iterators). # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Develop method for handling app deployment on heckel/jeckel # Can we use Bob Herlien to do some development work? # Get Mike M and Dave from (CenCOOS) and talk through SSDS for CenCOOS and getting them AUV data from MBARI. # Apply AJAX to web pages where performance problems exist ([http://code.jalenack.com/periodic/ Example]) # Fix Metadata links on CIMT pages to make them dynamic (or replace with different view, not XML) # Pull the MBARI ADCP page off the CIMT site # Look at [http://www.sensescape.com/scivis/index.html Visualization in Marine Science] # Look into [http://www.openmi.org OpenMI] # Turn on wrapper generator unit tests in continuous integration and process/display results # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## Make any direction plot (wind, heading, etc.) plot as points, not lines ## Put nominal lattitude and longitude in plot titles ## Have capability to turn on/off autoscale on plots and specify range # Get UI Design meetings going with the ops group and get portal prototype going # Ken Heller came by to ask to turn on more batch created plots for CIMT # Put link to ACE jar in web application # Ken also had another request to be able to see all variables plotted as thumbnails on one page, be able to select various plots and then have them show as full size together on another page (a merging of sorts for data). # Add pages for metadata editing (MetadataEditingNote) # SIAM``Raw``Data``Access Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement Query: Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location). This is specifically to support uploading the JARS for SIAM and linking them to Devices and Data``Producers(Deployments). # Implement Query: They would like to be able to store the driver class name that is linked to a Device so that they can have the user simply choose a device and it will start with that driver name. # Instead of using comand line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into having SSDS create "README" type files in the same location as certain Data``Containers (This could be a perl script bot) # Hold Brown Bag on architecture and uses. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional Standard``Variables, Standard``Units, Standard``Keywords, Standard``Domain, Standard``Reference``Scale, Device``Type, Resource``Type, Data``Producer``Group, Data``Container``Group and then notify the user of that change so they can change their source. # Remove Deployment info from PUCK XML (someday, probably for MSE). # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. # Write tests for Resource``BLOB->Object``Builder for byte array and verify that it is working correctly. # Send API's for SSDS ingest and Data``Stream Access to Data``Stream LOOKING group # Look at Mike Godin's Schema changes for AOSN # Can I embed the business logic documentation as Java``Doc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # See if I can use NT logins with Merlia driver in SSDS # Have the capability to turn off emails when resubmit is done. # Implmement generic way of sending data/XML ## Create application that reads in NetCDF file and generates XML to match our schema # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Cut down on JAR file bloat (happening in new arch) # Crawl all the old OASIS files and load the metadata into SSDS. # Meet with MMI to talk about pulling our SV and SU from ontologies. # XML with schemas declared not validating # Add mechanism that users can drop files of known type with all metadata in known format into a directory and SSDS will automatically register and store it. # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # Implement ingest socket -> JMS topic # Add QC flag to SSDS packets and match in database (SSDS_Data). # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # In Packet``Output``Manager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and Packet``Outputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add end of line terminator as separator in parsing packet records (not files) # Can I create something that will emulate reading from a serial port but is back by the SSDS data service? This would be done so that native applications that are used to reading from serial ports, can still do so. To be honest, this would be really cool, but seems fragile and unlikely. This started with the Nortek, so look at Nortek.no for what they called "Online applications" # Get LOBO data into SSDS # Get raw data porting from SSDS to LAS server working for Mooring data (Leverage Mike G's stuff). # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" data by storing comments with the data and returning that to user's upon request (think NetCDFs history) # Edit bad packets that are serialized to disk # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data) # Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Setup service that will calculate salinity automatically. # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Query: "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Create Process to create useable data file from inductive CTD string # Look at integrating Earth Google using services. # HOOVES Improvements: ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface # Look at NPS AUV Mission Planner as a way to interact with AUV data.Don Brutzman and Peter Flynn came to give a demo on this and it seems to be a possibility for a tool that can integrate and visualize modeling and collected data (as well as vehicle tracks). Just check Don's web address to find information ]]> 1179760 1212526 Fog-SSDS_Metadata h3. UpdateBot # Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) # Have updateBot crawl all resources and update contentLength if not specified. # Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. h3. Core (EAR) # Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds h3. Web applications # Verify PC02 plots are working after M0 turnaround # Develop admin application to edit all metadata objects and their relationships # Increase session timeout in Explorer h3. Clients # Migrate HOOVES to new architecture h3. Other # GoogleMaps/GoogleEarth/Worldwind integration # Build code to read data from Data``Files through the query interface (not just from packets). # Add capability in Explorer to export deployment XML template from Web to help in XML authoring. # Put UML diagram of data model on developer section of web app. # Implement a mechanism that updates the true end date/time on the Data``Container for data streams to show the latest data for each stream (This will affect MMI as well as the pages that show when last data was received, like SIAM raw data access page). This should also update the number of records and the stream should probably somehow reference the SQL data packets. # Configure Build/Code to make distributable and compatible with Mac (5 days) ## Put Copyright in all SSDS source code and zip up to put on web site # Put link to post processed data on base CIMT pages # Refactor navigation menus and scheme for SSDS site and post processed pages # Build web pages that all user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Build non-JMS mechanism for users to send data/metadata to SSDS. # Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Finish documenting data packet structure on web pages. # Verify (unit tests) that the RecordDescription level parse regular expression works # Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Follow up on PUCK configuration tool (ACE) # Verify that wrapper generator unit test are on during test target of build. # Put copyright in all code. # Export project to open source repository. # QC Pages ## show instruments removed for service or currently reflecting invalid data (e.g., cover left on) (John has notes of which) ### show expected turn-on date if known (note: would have to keep this up) ### add "Planned availability xxxx" to missing data and displays on website so appears more individually organized to users then just a generic "data not available" ## show processed QC data wherever available -- this is very valuable feature ## fix graphs that don't size correctly on Mark's machine (probably cross-platform browser problem) ## look for other cross-platform issues ## fix multispectral page heading (says "Backscatter/chlorophyll") ## on chlorophyll page, turbidity isn't presented (get plots working) ## add ISUS data (get plots working) ## add parsing of Medusa data where possible ## make first page to load the MSE Sites page rather than the Domains page ### Benthic nodes in Sites pages have some instruments not actually on node (check) ## Update anchor as-deployed position (MSE anchor is 36.218321° N -122.904986° W) ## Scale both axes equally on GPS plots that are scaled to data. ## Make all metadata (.xml) links work ### Be nice if metadata page were improved (why not put files in body of page?) ## Improve graph names on plots (particularly on power pages, n.b., "ADCnn") ## Correct the formatting of titles on plots (add spaces for clarity) ## Add portal data items (particularly Globalstar download durations) to data pages when available ## Delete Diagnostics page ## Add GPS plot showing location since deployment ## Change "Platform Environment" to "Platform Status" throughout ## need to show plots of summary data on these pages as soon as reasonably possible ## fix CTD post-processed plots that aren't updating (point to 20060502 instead of 20060503) ## Sure would be nice to see the real wind direction (must be computed) ## Be nice if QC plots knew to pick pen up when data is skipped ## Multiple variables on one axis is desirable (e.g., for Ed M) ## Consider deleting Sensor pages # Other Actions ## Mark: make links from MSE and MOOS web sites to SSDS MSE pages (update those sites) ## Mark: Review words at http://ssdspub.mbari.org:8080/mse/index.jsp ## Mark: send updated anchor location to Graybeal for watch circle plot ## Tom/SIAM: add SIAM infeastructure information to SSDS as SIAM packets ### Globalstar ppp info ### clock offsets ## Kevin: confirm SIAM Packets is the right way to add the SIAM infrastructure data ### provide mechanism for ISI ID or other data source ID ## TBD (McCann et al?): Aquadopp post-processing very desirable ## Tom/John: provide serial info to Paul for MSE GPS, discuss MTM3 GPS serial/ISI number issues w/him ### Tom: fix appropriate node to reflect correct GPS ISI ID ### Kevin: move data appropriately ### John: confirm primary GPS plots update correctly, put primary plots at top again ## Kevin: turn off fail-mail for missing sensors ## John: send missing sensor list ## Kent, Tom: look at the download times (vs. the real budget) and figure out what to do ## John: write XML translator for summary data (check w/Kevin first) ## John: note who is doing which post-processing somewhere (keep track) # Other Notes ## SOON post-processing data should be publicly displayed if possible, encourage scientists to do this. ### Raw data by itself cannot be interpreted or ground-truthed by anyone but instrument builder. ### MSE policy is for all MSE data to be publicly available. ## Enthusiasm expressed for post-processed data ## Enthusiasm expressed for ability to drill down to deployed sensors and their data using Explorer ## The existing diagnostic info on sensor pages (e.g., on LWR) is just fine ## Desire in long term for common 'status reporting' system for instruments ## Need to clean up presentation of left navigation on main SSDS page ## Need to add rest of SSDS web page content to main SSDS pages ## Being able to search and sort instruments by current location/deployment is extremely valuable # Refactor/overhaul chart creation application ## Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) ## Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) ## Setup Device``QCPlot``Creator to create plots with multiple lines on one chart (and separate axes). # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # Implement Query "I want the Air Temperature (Standard``Variable.name) from the Data``Producer named "M1" from such and such to such and such a time." # Query: Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data``Producer of type Deployment? # Query: Given a deployment, can I walk up the parent tree to look for a parent with a certain role? # Figure out how we will integrate data files offloaded from instruments when they are recovered. # Site Based Pages for MSE (4 days) # Possibility of direct support of ESB work with NCSA/LOOKING h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Verify Done # Add boolean to all query methods for returning full object graphs. # Implement methods to returns counts on queries. # Add capability of specifying what to sort results by (property name) in the query methods. # Returned DataContainer collections should be sorted by start date as default # Query for DataContainer by DataContainerGroup # Make sure XML dates that are coming out of XML``Builder/Object``Builder cycle can be parsed by the XML``Date``Format. # Check Object Builder/XML builder to make sure it handles URI/URL/Uri``String/DODS``Url/X/Y/Z``offsets correctly. # Make sure object Schema/Object``Builder/XML``Builder only use one Data``Container/Data``Producer for output, input, consumer, tags. h3. Completed Tasks # Look at AUVCTD portal code and make sure it survives an SSDS restart # Finish architecture migration: ## Finish UpdateBot ### Source/Destination tracking for flagging NetCDF creation ### Email after processing ## Change unique key definition on Standard``Variable code ## Rebuild code, deploy on ssdsdevpc and let it build database on Fog. ## Fix/Run DTS to accomodate the new alternate primary key on Standard``Variable to be name and namespaceUri and copy data from SSDS on Solstice to SSDS_Metadata on Fog. ## Rebuild code base, deploy to new-ssds and let code build DB on Solstice SSDS_Metadata. (TURN OFF MESSAGING EMAIL TILL AFTER MIKE DOES STANDARD VARIABLE UPDATES). ## Rebuild code base, redeploy with Create DB off. ## Change DTS job from Fog->SSDS_Metadata to Solstice->SSDS_Metadata to match the one used for SSDSDevPC and run. ## Startup new-ssds jboss. ## Run all MSE Urls. ## Have Mike run Stanard``Variable updates. ## Rebuild code and deploy with messaging turned back on. ## Verify all data of interest for MSE is being plotted ## Move MTM3 web apps to same form as MSE. ## New Device creation/editing page ## New Person creation/editing page ## Point old jsp's to new jsp's ## Point old Get``Original``Data``Servlet (ssds.shore and ssdpsub) to new Get``Original``Data``Servlet ## Migrate HOOVES. ## Shut off ruminate on predator. ## Shut off cimt on predator. ## Undeploy all SSDS stuff except for access.war that has forwards in it. ## Verify what Luis is using for Salinity is just Get``Original``Data``Servlet. ## Leave for a bit, but remove SSDS from solstice. ## Remove SSDS from Fog. ## Clean out do fresh installation on prey (retask). ## Turn off replication of data from tornado to ssdspub ## Remove SSDS data database from SSDSPub and all that should remain is a few .war files pointing to new-ssds. ## Ensure M0 PC02 Plots working (M0 Turnaround) # Break up deployments in metadata for Mooring into pre-deployment and deployment (M0 is the real issue) (no progress, re-iterated by Francisco) # In build, fix so that the jsf-lib directory is removed and install jstl.jar & standard.jar # What is the official MSE name? (MTM Meeting next week). # Find NetCDF Files for CIMT data and send URLs to Brian Fulfrost # Web page access to data from final ADCP netcdf file (Mike M completed) # Find location and put XML schemas (and associated DTD's out on the public web). Also include that as part of the build process (http://new-ssds.mbari.org/ssds-docs/xml/schema/SSDS_Metadata.xsd) # Add uri attribute to Standard``Variable (Valid URI) # Alternate primary key on SV (name, uri) # Can we implement a method on Process``Run``Access that can do cascade deletes? (done in new architecture on Data``Producer``DAO and is called "deepDelete") # "I'm calling Process``Run``Access.update() with additional inputs and they are not getting added. (new architecture does this) # Get the graphs on the CIMT web site working with device 1480. # The Time/Date box in the SIAM``Raw``Data``Access``Page states it is PST, but that is no longer true since we moved to having the Jboss instance running in GMT. # Implement findWithinDateRange() (Mike M) method is called findByDateRangeAndName and can be invoked with null name to just search by date. # Implement findByDataProducerGroupName() [remove findByLikeDataProducerGroupName() & other ..like.. methods] (Mike M) # Return count methods, e.g. countOfFindByDataProducerType() (Mike M) (ongoing, but done for the ones listed here) # findByLikeNameWithinTimeAndWithinGeospatialCube() [remove the ..like.. & use exactMatch boolean] (Mike M) (Done and method is called findByNameAndTimeAndGeospatialCube). # Would these countOf...() methods be easy to implement? They would sure be handy to have from a user interface perspective. (Mike M) (Done for the ones in this completed section, but still an ongoing effort). # Add Resouce``BLOB to Resource (Andrew). # Add Keywords to Data``Container through servlet (Andrew) (This should work now, but not really tested) # Query for Data``Container by Keyword (Andrew) (Done, but I am going to refactor to implement one query method like I did in Data``Producer``DAO). # Query for Data``Container by Lat/Lon and by time (Andrew) (Done, but I am going to refactor to implement one query method like I did in Data``Producer``DAO). # Object/XML Builder done for new architecture. # Metadata``Access``Servlet updated for new architecture # Fixed bug in Metadata``Factory to return primitive classes when primitive text is passed in. # Fixed DTS job to copy from old database to new on Fog (1/11/2006) # Change SSDS so it can handle new Summary``Packet from SIAM. No change was necessary as Tom is exporting Summary``Packets as Sensor``Data``Packets # Figure out if we need to submit any text for SSDS Project for Annual Report (due 2/1) # Get Andrew's Metadata``Access``Servlet documentation up on servlet and/or ssds web application # Write up new architecture and migration path for new machine and architecture deployment. # Simplify operation tasks (startup, shutdown, health, etc.) # Document requirements for data access for MSE and get those and schedule to Keith ## Talk to Charlie, Jim Barry, Bill Ussler, Rendy, Chris Lovera about how they process data from the instruments currently. ## Create list of UI goals for this (and beyond) and figure out taks/skills for that. # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## On plots, link sensor manufacturer and model and put on plot itself (if possible) ## If min/max value exist for record variables, clip generated plots to those values. Also can use the new displayMin/displayMax variables where data gaps are) # Look into Ken's suggestion about ID's for deployment. From his email {noformat} All, When looking at the raw data in SSDS is can be very difficult to figure out what is what mooring and will get even more so as time goes by. This email is an attempt to get discussions going on how to deal with eliminating the confusion. All instruments, including controller cans can (and have been) be swapped during the deployment. It would be nice to have an ID for each deployment i.e. 2004 M0, 2005 M0, 2005 M2.If this ID were set up in the can as the parent ID just before deployment (i.e. aboard ship on the way out), then the problem of trying to separate out test data from deployed data would be easier to manage. It should also help resolve the issue of having latitude and longitude data in pucks. Instead the mooring ID would have the location associated with it. Regards, Ken Heller {noformat} # Implement Query: Find all instruments currently deployed on a "deployment" # Implement Query: They want to be able to selectively view only Metdata``Packets (Record``Type) and find only the most recent one. # Implement Query: They want a way to know what is currently deployed (user interface). # Fix Java``Script bug in SIAM Raw Data Access Page: This is a bug that Andy Hamilton showed me. When you select start to end time option, the get data, then click on back, the Javascript diables some of the time selection fields, which then invalidate the query and if you try to get the data again, you get a Null``Pointer``Exception. # Follow up with ops group with meeting on ID's, device, sensors, etc. and clean up device DB in SSDS # Add way to lookup instruments by platform on the device table # Merge access method (SIAM``Raw``Data``Access and Get``Original``Data``Servlet) to have same options and same output, web page simply forms the URL. An email {noformat} Hi Kevin, I'm noticing that the GetOriginalDataServlet app seems to be adding an additional CR/LF between the ISUS samples (each sample has four data records) when I snarf it directly as opposed to displaying it via a browser. See the URL below to see what I'm describing. It won't be a problem as far as I'm concerned just something I noticed. http://www2.mbari.org/coletti/m0_isus.cgi Luke And another Ah Ha, look what happens when you set noHTMLHeader=0 (URL below)... http://ssds.shore.mbari.org:8080/access/GetOriginalDataServlet?platformID=1299&metadataID=0 &recordTypeID=1&isi=0&noHTMLHeader=0&deviceID=1268&startDateTime=20040603.223400&endDateTime=20040606.223400, Luke {noformat} # Implmement basic metadata/data QC internal to SSDS (watchdog) # Refactor Ruminate in new architecture # When information in XML is inconsistent (device ID) with that of packet, somebody should be notified. # If device comes in with new ID, it will be updated/persisted, should not' # Performance investigation of services (sort of ongoing process) # The XML``Metadata``Tracker does not look like it is working properly (especially on AUV files) in the ruminate process # Andrew metioned that the depths (Data``Container) are rounding. # Boolean not setting on incoming Metadata``Access``Servlet query (verify this is fixed in new architecture, then move to completed) # Connect up raw stream access to the SQL database stream storage # Add parameter that the user can ask for the last X number of records. # Can we setup the Record``Description so that it can define a regular expression that can parse the whole record into variables? # Can we have a service that will automatically take NEMA strings from GPS's and convert them to a more useful format? # Implement Time``Indexed``Net``CDF``Access class # Crawl all data stream files and republish data to SQL topic to populate DB with device stream data. # Get NetCDF Watchdog going to keep NetCDF files created for those Data``Containers that ask for them. # Get XML Defined for the Nortek binary case (variable binary length) # Data servlet variables out of order and no flag for units # Fix Bob's Code so it queries by deployment name, not device name then change 1305 and 1414 device names back to OASIS Bouy.Then go back and clean up device list per Paul C. (Remove some of the M1/M2 designations). Remove devices 1307 and 1308 from database. # Re-run all loads for AUVCTD data onto the production machine to make sure it is all there(?) # When SSDSPUB tries to read from dods.mbari.org, it times out. # Verify what Luis is using for Salinity is just Get``Original``Data``Servlet. h3. Future Tasks (Shelved) # Give Nancy Barr a base URL and she can have one of her crawlers check the links # Look at porting code to J2SE 5 (Generics, Iterators). # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Develop method for handling app deployment on heckel/jeckel # Can we use Bob Herlien to do some development work? # Get Mike M and Dave from (CenCOOS) and talk through SSDS for CenCOOS and getting them AUV data from MBARI. # Apply AJAX to web pages where performance problems exist ([http://code.jalenack.com/periodic/ Example]) # Fix Metadata links on CIMT pages to make them dynamic (or replace with different view, not XML) # Pull the MBARI ADCP page off the CIMT site # Look at [http://www.sensescape.com/scivis/index.html Visualization in Marine Science] # Look into [http://www.openmi.org OpenMI] # Turn on wrapper generator unit tests in continuous integration and process/display results # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## Make any direction plot (wind, heading, etc.) plot as points, not lines ## Put nominal lattitude and longitude in plot titles ## Have capability to turn on/off autoscale on plots and specify range # Get UI Design meetings going with the ops group and get portal prototype going # Ken Heller came by to ask to turn on more batch created plots for CIMT # Put link to ACE jar in web application # Ken also had another request to be able to see all variables plotted as thumbnails on one page, be able to select various plots and then have them show as full size together on another page (a merging of sorts for data). # Add pages for metadata editing (MetadataEditingNote) # SIAM``Raw``Data``Access Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement Query: Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location). This is specifically to support uploading the JARS for SIAM and linking them to Devices and Data``Producers(Deployments). # Implement Query: They would like to be able to store the driver class name that is linked to a Device so that they can have the user simply choose a device and it will start with that driver name. # Instead of using comand line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into having SSDS create "README" type files in the same location as certain Data``Containers (This could be a perl script bot) # Hold Brown Bag on architecture and uses. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional Standard``Variables, Standard``Units, Standard``Keywords, Standard``Domain, Standard``Reference``Scale, Device``Type, Resource``Type, Data``Producer``Group, Data``Container``Group and then notify the user of that change so they can change their source. # Remove Deployment info from PUCK XML (someday, probably for MSE). # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. # Write tests for Resource``BLOB->Object``Builder for byte array and verify that it is working correctly. # Send API's for SSDS ingest and Data``Stream Access to Data``Stream LOOKING group # Look at Mike Godin's Schema changes for AOSN # Can I embed the business logic documentation as Java``Doc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # See if I can use NT logins with Merlia driver in SSDS # Have the capability to turn off emails when resubmit is done. # Implmement generic way of sending data/XML ## Create application that reads in NetCDF file and generates XML to match our schema # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Cut down on JAR file bloat (happening in new arch) # Crawl all the old OASIS files and load the metadata into SSDS. # Meet with MMI to talk about pulling our SV and SU from ontologies. # XML with schemas declared not validating # Add mechanism that users can drop files of known type with all metadata in known format into a directory and SSDS will automatically register and store it. # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # Implement ingest socket -> JMS topic # Add QC flag to SSDS packets and match in database (SSDS_Data). # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # In Packet``Output``Manager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and Packet``Outputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add end of line terminator as separator in parsing packet records (not files) # Can I create something that will emulate reading from a serial port but is back by the SSDS data service? This would be done so that native applications that are used to reading from serial ports, can still do so. To be honest, this would be really cool, but seems fragile and unlikely. This started with the Nortek, so look at Nortek.no for what they called "Online applications" # Get LOBO data into SSDS # Get raw data porting from SSDS to LAS server working for Mooring data (Leverage Mike G's stuff). # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" data by storing comments with the data and returning that to user's upon request (think NetCDFs history) # Edit bad packets that are serialized to disk # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data) # Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Setup service that will calculate salinity automatically. # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Query: "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Create Process to create useable data file from inductive CTD string # Look at integrating Earth Google using services. # HOOVES Improvements: ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface # Look at NPS AUV Mission Planner as a way to interact with AUV data.Don Brutzman and Peter Flynn came to give a demo on this and it seems to be a possibility for a tool that can integrate and visualize modeling and collected data (as well as vehicle tracks). Just check Don's web address to find information ]]> 1179759 1212524 SSDS_Metadata to Solstice->SSDS_Metadata to match the one used for SSDSDevPC and run. ## Startup new-ssds jboss. ## Run all MSE Urls. ## Have Mike run Stanard``Variable updates. ## Rebuild code and deploy with messaging turned back on. ## Verify all data of interest for MSE is being plotted ## Move MTM3 web apps to same form as MSE. ## New Device creation/editing page ## New Person creation/editing page ## Point old jsp's to new jsp's ## Point old Get``Original``Data``Servlet (ssds.shore and ssdpsub) to new Get``Original``Data``Servlet ## Migrate HOOVES. ## Shut off ruminate on predator. ## Shut off cimt on predator. ## Undeploy all SSDS stuff except for access.war that has forwards in it. ## Verify what Luis is using for Salinity is just Get``Original``Data``Servlet. ## Leave for a bit, but remove SSDS from solstice. ## Remove SSDS from Fog. ## Clean out do fresh installation on prey (retask). ## Turn off replication of data from tornado to ssdspub ## Remove SSDS data database from SSDSPub and all that should remain is a few .war files pointing to new-ssds. ## Ensure M0 PC02 Plots working (M0 Turnaround) # Break up deployments in metadata for Mooring into pre-deployment and deployment (M0 is the real issue) (no progress, re-iterated by Francisco) # In build, fix so that the jsf-lib directory is removed and install jstl.jar & standard.jar # What is the official MSE name? (MTM Meeting next week). # Find NetCDF Files for CIMT data and send URLs to Brian Fulfrost # Web page access to data from final ADCP netcdf file (Mike M completed) # Find location and put XML schemas (and associated DTD's out on the public web). Also include that as part of the build process (http://new-ssds.mbari.org/ssds-docs/xml/schema/SSDS_Metadata.xsd) # Add uri attribute to Standard``Variable (Valid URI) # Alternate primary key on SV (name, uri) # Can we implement a method on Process``Run``Access that can do cascade deletes? (done in new architecture on Data``Producer``DAO and is called "deepDelete") # "I'm calling Process``Run``Access.update() with additional inputs and they are not getting added. (new architecture does this) # Get the graphs on the CIMT web site working with device 1480. # The Time/Date box in the SIAM``Raw``Data``Access``Page states it is PST, but that is no longer true since we moved to having the Jboss instance running in GMT. # Implement findWithinDateRange() (Mike M) method is called findByDateRangeAndName and can be invoked with null name to just search by date. # Implement findByDataProducerGroupName() [remove findByLikeDataProducerGroupName() & other ..like.. methods] (Mike M) # Return count methods, e.g. countOfFindByDataProducerType() (Mike M) (ongoing, but done for the ones listed here) # findByLikeNameWithinTimeAndWithinGeospatialCube() [remove the ..like.. & use exactMatch boolean] (Mike M) (Done and method is called findByNameAndTimeAndGeospatialCube). # Would these countOf...() methods be easy to implement? They would sure be handy to have from a user interface perspective. (Mike M) (Done for the ones in this completed section, but still an ongoing effort). # Add Resouce``BLOB to Resource (Andrew). # Add Keywords to Data``Container through servlet (Andrew) (This should work now, but not really tested) # Query for Data``Container by Keyword (Andrew) (Done, but I am going to refactor to implement one query method like I did in Data``Producer``DAO). # Query for Data``Container by Lat/Lon and by time (Andrew) (Done, but I am going to refactor to implement one query method like I did in Data``Producer``DAO). # Object/XML Builder done for new architecture. # Metadata``Access``Servlet updated for new architecture # Fixed bug in Metadata``Factory to return primitive classes when primitive text is passed in. # Fixed DTS job to copy from old database to new on Fog (1/11/2006) # Change SSDS so it can handle new Summary``Packet from SIAM. No change was necessary as Tom is exporting Summary``Packets as Sensor``Data``Packets # Figure out if we need to submit any text for SSDS Project for Annual Report (due 2/1) # Get Andrew's Metadata``Access``Servlet documentation up on servlet and/or ssds web application # Write up new architecture and migration path for new machine and architecture deployment. # Simplify operation tasks (startup, shutdown, health, etc.) # Document requirements for data access for MSE and get those and schedule to Keith ## Talk to Charlie, Jim Barry, Bill Ussler, Rendy, Chris Lovera about how they process data from the instruments currently. ## Create list of UI goals for this (and beyond) and figure out taks/skills for that. # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## On plots, link sensor manufacturer and model and put on plot itself (if possible) ## If min/max value exist for record variables, clip generated plots to those values. Also can use the new displayMin/displayMax variables where data gaps are) # Look into Ken's suggestion about ID's for deployment. From his email {noformat} All, When looking at the raw data in SSDS is can be very difficult to figure out what is what mooring and will get even more so as time goes by. This email is an attempt to get discussions going on how to deal with eliminating the confusion. All instruments, including controller cans can (and have been) be swapped during the deployment. It would be nice to have an ID for each deployment i.e. 2004 M0, 2005 M0, 2005 M2.If this ID were set up in the can as the parent ID just before deployment (i.e. aboard ship on the way out), then the problem of trying to separate out test data from deployed data would be easier to manage. It should also help resolve the issue of having latitude and longitude data in pucks. Instead the mooring ID would have the location associated with it. Regards, Ken Heller {noformat} # Implement Query: Find all instruments currently deployed on a "deployment" # Implement Query: They want to be able to selectively view only Metdata``Packets (Record``Type) and find only the most recent one. # Implement Query: They want a way to know what is currently deployed (user interface). # Fix Java``Script bug in SIAM Raw Data Access Page: This is a bug that Andy Hamilton showed me. When you select start to end time option, the get data, then click on back, the Javascript diables some of the time selection fields, which then invalidate the query and if you try to get the data again, you get a Null``Pointer``Exception. # Follow up with ops group with meeting on ID's, device, sensors, etc. and clean up device DB in SSDS # Add way to lookup instruments by platform on the device table # Merge access method (SIAM``Raw``Data``Access and Get``Original``Data``Servlet) to have same options and same output, web page simply forms the URL. An email {noformat} Hi Kevin, I'm noticing that the GetOriginalDataServlet app seems to be adding an additional CR/LF between the ISUS samples (each sample has four data records) when I snarf it directly as opposed to displaying it via a browser. See the URL below to see what I'm describing. It won't be a problem as far as I'm concerned just something I noticed. http://www2.mbari.org/coletti/m0_isus.cgi Luke And another Ah Ha, look what happens when you set noHTMLHeader=0 (URL below)... http://ssds.shore.mbari.org:8080/access/GetOriginalDataServlet?platformID=1299&metadataID=0 &recordTypeID=1&isi=0&noHTMLHeader=0&deviceID=1268&startDateTime=20040603.223400&endDateTime=20040606.223400, Luke {noformat} # Implmement basic metadata/data QC internal to SSDS (watchdog) # Refactor Ruminate in new architecture # When information in XML is inconsistent (device ID) with that of packet, somebody should be notified. # If device comes in with new ID, it will be updated/persisted, should not' # Performance investigation of services (sort of ongoing process) # The XML``Metadata``Tracker does not look like it is working properly (especially on AUV files) in the ruminate process # Andrew metioned that the depths (Data``Container) are rounding. # Boolean not setting on incoming Metadata``Access``Servlet query (verify this is fixed in new architecture, then move to completed) # Connect up raw stream access to the SQL database stream storage # Add parameter that the user can ask for the last X number of records. # Can we setup the Record``Description so that it can define a regular expression that can parse the whole record into variables? # Can we have a service that will automatically take NEMA strings from GPS's and convert them to a more useful format? # Implement Time``Indexed``Net``CDF``Access class # Crawl all data stream files and republish data to SQL topic to populate DB with device stream data. # Get NetCDF Watchdog going to keep NetCDF files created for those Data``Containers that ask for them. # Get XML Defined for the Nortek binary case (variable binary length) # Data servlet variables out of order and no flag for units # Fix Bob's Code so it queries by deployment name, not device name then change 1305 and 1414 device names back to OASIS Bouy.Then go back and clean up device list per Paul C. (Remove some of the M1/M2 designations). Remove devices 1307 and 1308 from database. # Re-run all loads for AUVCTD data onto the production machine to make sure it is all there(?) # When SSDSPUB tries to read from dods.mbari.org, it times out. h3. Future Tasks (Shelved) # Give Nancy Barr a base URL and she can have one of her crawlers check the links # Look at porting code to J2SE 5 (Generics, Iterators). # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Develop method for handling app deployment on heckel/jeckel # Can we use Bob Herlien to do some development work? # Get Mike M and Dave from (CenCOOS) and talk through SSDS for CenCOOS and getting them AUV data from MBARI. # Apply AJAX to web pages where performance problems exist ([http://code.jalenack.com/periodic/ Example]) # Fix Metadata links on CIMT pages to make them dynamic (or replace with different view, not XML) # Pull the MBARI ADCP page off the CIMT site # Look at [http://www.sensescape.com/scivis/index.html Visualization in Marine Science] # Look into [http://www.openmi.org OpenMI] # Turn on wrapper generator unit tests in continuous integration and process/display results # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## Make any direction plot (wind, heading, etc.) plot as points, not lines ## Put nominal lattitude and longitude in plot titles ## Have capability to turn on/off autoscale on plots and specify range # Get UI Design meetings going with the ops group and get portal prototype going # Ken Heller came by to ask to turn on more batch created plots for CIMT # Put link to ACE jar in web application # Ken also had another request to be able to see all variables plotted as thumbnails on one page, be able to select various plots and then have them show as full size together on another page (a merging of sorts for data). # Add pages for metadata editing (MetadataEditingNote) # SIAM``Raw``Data``Access Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement Query: Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location). This is specifically to support uploading the JARS for SIAM and linking them to Devices and Data``Producers(Deployments). # Implement Query: They would like to be able to store the driver class name that is linked to a Device so that they can have the user simply choose a device and it will start with that driver name. # Instead of using comand line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into having SSDS create "README" type files in the same location as certain Data``Containers (This could be a perl script bot) # Hold Brown Bag on architecture and uses. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional Standard``Variables, Standard``Units, Standard``Keywords, Standard``Domain, Standard``Reference``Scale, Device``Type, Resource``Type, Data``Producer``Group, Data``Container``Group and then notify the user of that change so they can change their source. # Remove Deployment info from PUCK XML (someday, probably for MSE). # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. # Write tests for Resource``BLOB->Object``Builder for byte array and verify that it is working correctly. # Send API's for SSDS ingest and Data``Stream Access to Data``Stream LOOKING group # Look at Mike Godin's Schema changes for AOSN # Can I embed the business logic documentation as Java``Doc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # See if I can use NT logins with Merlia driver in SSDS # Have the capability to turn off emails when resubmit is done. # Implmement generic way of sending data/XML ## Create application that reads in NetCDF file and generates XML to match our schema # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Cut down on JAR file bloat (happening in new arch) # Crawl all the old OASIS files and load the metadata into SSDS. # Meet with MMI to talk about pulling our SV and SU from ontologies. # XML with schemas declared not validating # Add mechanism that users can drop files of known type with all metadata in known format into a directory and SSDS will automatically register and store it. # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # Implement ingest socket -> JMS topic # Add QC flag to SSDS packets and match in database (SSDS_Data). # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # In Packet``Output``Manager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and Packet``Outputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add end of line terminator as separator in parsing packet records (not files) # Can I create something that will emulate reading from a serial port but is back by the SSDS data service? This would be done so that native applications that are used to reading from serial ports, can still do so. To be honest, this would be really cool, but seems fragile and unlikely. This started with the Nortek, so look at Nortek.no for what they called "Online applications" # Get LOBO data into SSDS # Get raw data porting from SSDS to LAS server working for Mooring data (Leverage Mike G's stuff). # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" data by storing comments with the data and returning that to user's upon request (think NetCDFs history) # Edit bad packets that are serialized to disk # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data) # Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Setup service that will calculate salinity automatically. # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Query: "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Create Process to create useable data file from inductive CTD string # Look at integrating Earth Google using services. # HOOVES Improvements: ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface # Look at NPS AUV Mission Planner as a way to interact with AUV data.Don Brutzman and Peter Flynn came to give a demo on this and it seems to be a possibility for a tool that can integrate and visualize modeling and collected data (as well as vehicle tracks). Just check Don's web address to find information ]]> 1179757 1212523 SSDS_Metadata to Solstice->SSDS_Metadata to match the one used for SSDSDevPC and run. ## Startup new-ssds jboss. ## Run all MSE Urls. ## Have Mike run Stanard``Variable updates. ## Rebuild code and deploy with messaging turned back on. ## Verify all data of interest for MSE is being plotted ## Move MTM3 web apps to same form as MSE. ## New Device creation/editing page ## New Person creation/editing page ## Point old jsp's to new jsp's ## Point old Get``Original``Data``Servlet (ssds.shore and ssdpsub) to new Get``Original``Data``Servlet ## Migrate HOOVES. ## Shut off ruminate on predator. ## Shut off cimt on predator. ## Undeploy all SSDS stuff except for access.war that has forwards in it. ## Verify what Luis is using for Salinity is just Get``Original``Data``Servlet. ## Leave for a bit, but remove SSDS from solstice. ## Remove SSDS from Fog. ## Clean out do fresh installation on prey (retask). ## Turn off replication of data from tornado to ssdspub ## Remove SSDS data database from SSDSPub and all that should remain is a few .war files pointing to new-ssds. ## Ensure M0 PC02 Plots working (M0 Turnaround) # Break up deployments in metadata for Mooring into pre-deployment and deployment (M0 is the real issue) (no progress, re-iterated by Francisco) # In build, fix so that the jsf-lib directory is removed and install jstl.jar & standard.jar # What is the official MSE name? (MTM Meeting next week). # Find NetCDF Files for CIMT data and send URLs to Brian Fulfrost # Web page access to data from final ADCP netcdf file (Mike M completed) # Find location and put XML schemas (and associated DTD's out on the public web). Also include that as part of the build process (http://new-ssds.mbari.org/ssds-docs/xml/schema/SSDS_Metadata.xsd) # Add uri attribute to Standard``Variable (Valid URI) # Alternate primary key on SV (name, uri) # Can we implement a method on Process``Run``Access that can do cascade deletes? (done in new architecture on Data``Producer``DAO and is called "deepDelete") # "I'm calling Process``Run``Access.update() with additional inputs and they are not getting added. (new architecture does this) # Get the graphs on the CIMT web site working with device 1480. # The Time/Date box in the SIAM``Raw``Data``Access``Page states it is PST, but that is no longer true since we moved to having the Jboss instance running in GMT. # Implement findWithinDateRange() (Mike M) method is called findByDateRangeAndName and can be invoked with null name to just search by date. # Implement findByDataProducerGroupName() [remove findByLikeDataProducerGroupName() & other ..like.. methods] (Mike M) # Return count methods, e.g. countOfFindByDataProducerType() (Mike M) (ongoing, but done for the ones listed here) # findByLikeNameWithinTimeAndWithinGeospatialCube() [remove the ..like.. & use exactMatch boolean] (Mike M) (Done and method is called findByNameAndTimeAndGeospatialCube). # Would these countOf...() methods be easy to implement? They would sure be handy to have from a user interface perspective. (Mike M) (Done for the ones in this completed section, but still an ongoing effort). # Add Resouce``BLOB to Resource (Andrew). # Add Keywords to Data``Container through servlet (Andrew) (This should work now, but not really tested) # Query for Data``Container by Keyword (Andrew) (Done, but I am going to refactor to implement one query method like I did in Data``Producer``DAO). # Query for Data``Container by Lat/Lon and by time (Andrew) (Done, but I am going to refactor to implement one query method like I did in Data``Producer``DAO). # Object/XML Builder done for new architecture. # Metadata``Access``Servlet updated for new architecture # Fixed bug in Metadata``Factory to return primitive classes when primitive text is passed in. # Fixed DTS job to copy from old database to new on Fog (1/11/2006) # Change SSDS so it can handle new Summary``Packet from SIAM. No change was necessary as Tom is exporting Summary``Packets as Sensor``Data``Packets # Figure out if we need to submit any text for SSDS Project for Annual Report (due 2/1) # Get Andrew's Metadata``Access``Servlet documentation up on servlet and/or ssds web application # Write up new architecture and migration path for new machine and architecture deployment. # Simplify operation tasks (startup, shutdown, health, etc.) # Document requirements for data access for MSE and get those and schedule to Keith ## Talk to Charlie, Jim Barry, Bill Ussler, Rendy, Chris Lovera about how they process data from the instruments currently. ## Create list of UI goals for this (and beyond) and figure out taks/skills for that. # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## On plots, link sensor manufacturer and model and put on plot itself (if possible) ## If min/max value exist for record variables, clip generated plots to those values. Also can use the new displayMin/displayMax variables where data gaps are) # Look into Ken's suggestion about ID's for deployment. From his email {noformat} All, When looking at the raw data in SSDS is can be very difficult to figure out what is what mooring and will get even more so as time goes by. This email is an attempt to get discussions going on how to deal with eliminating the confusion. All instruments, including controller cans can (and have been) be swapped during the deployment. It would be nice to have an ID for each deployment i.e. 2004 M0, 2005 M0, 2005 M2.If this ID were set up in the can as the parent ID just before deployment (i.e. aboard ship on the way out), then the problem of trying to separate out test data from deployed data would be easier to manage. It should also help resolve the issue of having latitude and longitude data in pucks. Instead the mooring ID would have the location associated with it. Regards, Ken Heller {noformat} # Implement Query: Find all instruments currently deployed on a "deployment" # Implement Query: They want to be able to selectively view only Metdata``Packets (Record``Type) and find only the most recent one. # Implement Query: They want a way to know what is currently deployed (user interface). # Fix Java``Script bug in SIAM Raw Data Access Page: This is a bug that Andy Hamilton showed me. When you select start to end time option, the get data, then click on back, the Javascript diables some of the time selection fields, which then invalidate the query and if you try to get the data again, you get a Null``Pointer``Exception. # Follow up with ops group with meeting on ID's, device, sensors, etc. and clean up device DB in SSDS # Add way to lookup instruments by platform on the device table # Merge access method (SIAM``Raw``Data``Access and Get``Original``Data``Servlet) to have same options and same output, web page simply forms the URL. An email {noformat} Hi Kevin, I'm noticing that the GetOriginalDataServlet app seems to be adding an additional CR/LF between the ISUS samples (each sample has four data records) when I snarf it directly as opposed to displaying it via a browser. See the URL below to see what I'm describing. It won't be a problem as far as I'm concerned just something I noticed. http://www2.mbari.org/coletti/m0_isus.cgi Luke And another Ah Ha, look what happens when you set noHTMLHeader=0 (URL below)... http://ssds.shore.mbari.org:8080/access/GetOriginalDataServlet?platformID=1299&metadataID=0 &recordTypeID=1&isi=0&noHTMLHeader=0&deviceID=1268&startDateTime=20040603.223400&endDateTime=20040606.223400, Luke {noformat} # Implmement basic metadata/data QC internal to SSDS (watchdog) # Refactor Ruminate in new architecture # When information in XML is inconsistent (device ID) with that of packet, somebody should be notified. # If device comes in with new ID, it will be updated/persisted, should not' # Performance investigation of services (sort of ongoing process) # The XML``Metadata``Tracker does not look like it is working properly (especially on AUV files) in the ruminate process # Andrew metioned that the depths (Data``Container) are rounding. # Boolean not setting on incoming Metadata``Access``Servlet query (verify this is fixed in new architecture, then move to completed) # Connect up raw stream access to the SQL database stream storage # Add parameter that the user can ask for the last X number of records. # Can we setup the Record``Description so that it can define a regular expression that can parse the whole record into variables? # Can we have a service that will automatically take NEMA strings from GPS's and convert them to a more useful format? # Implement Time``Indexed``Net``CDF``Access class # Crawl all data stream files and republish data to SQL topic to populate DB with device stream data. # Get NetCDF Watchdog going to keep NetCDF files created for those Data``Containers that ask for them. # Get XML Defined for the Nortek binary case (variable binary length) # Data servlet variables out of order and no flag for units # Fix Bob's Code so it queries by deployment name, not device name then change 1305 and 1414 device names back to OASIS Bouy.Then go back and clean up device list per Paul C. (Remove some of the M1/M2 designations). Remove devices 1307 and 1308 from database. # Re-run all loads for AUVCTD data onto the production machine to make sure it is all there(?) # When SSDSPUB tries to read from dods.mbari.org, it times out. h3. Future Tasks (Shelved) # Give Nancy Barr a base URL and she can have one of her crawlers check the links # Look at porting code to J2SE 5 (Generics, Iterators). # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Develop method for handling app deployment on heckel/jeckel # Can we use Bob Herlien to do some development work? # Get Mike M and Dave from (CenCOOS) and talk through SSDS for CenCOOS and getting them AUV data from MBARI. # Apply AJAX to web pages where performance problems exist ([http://code.jalenack.com/periodic/ Example]) # Fix Metadata links on CIMT pages to make them dynamic (or replace with different view, not XML) # Pull the MBARI ADCP page off the CIMT site # Look at [http://www.sensescape.com/scivis/index.html Visualization in Marine Science] # Look into [http://www.openmi.org OpenMI] # Turn on wrapper generator unit tests in continuous integration and process/display results # Refactor/overhaul CIMT chart creation application to allow users to configure their own offline chart creation ## Make any direction plot (wind, heading, etc.) plot as points, not lines ## Put nominal lattitude and longitude in plot titles ## Have capability to turn on/off autoscale on plots and specify range # Get UI Design meetings going with the ops group and get portal prototype going # Ken Heller came by to ask to turn on more batch created plots for CIMT # Put link to ACE jar in web application # Ken also had another request to be able to see all variables plotted as thumbnails on one page, be able to select various plots and then have them show as full size together on another page (a merging of sorts for data). # Add pages for metadata editing (MetadataEditingNote) # SIAM``Raw``Data``Access Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement Query: Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location). This is specifically to support uploading the JARS for SIAM and linking them to Devices and Data``Producers(Deployments). # Implement Query: They would like to be able to store the driver class name that is linked to a Device so that they can have the user simply choose a device and it will start with that driver name. # Instead of using comand line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into having SSDS create "README" type files in the same location as certain Data``Containers (This could be a perl script bot) # Hold Brown Bag on architecture and uses. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional Standard``Variables, Standard``Units, Standard``Keywords, Standard``Domain, Standard``Reference``Scale, Device``Type, Resource``Type, Data``Producer``Group, Data``Container``Group and then notify the user of that change so they can change their source. # Remove Deployment info from PUCK XML (someday, probably for MSE). # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. # Write tests for Resource``BLOB->Object``Builder for byte array and verify that it is working correctly. # Send API's for SSDS ingest and Data``Stream Access to Data``Stream LOOKING group # Look at Mike Godin's Schema changes for AOSN # Can I embed the business logic documentation as Java``Doc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # See if I can use NT logins with Merlia driver in SSDS # Have the capability to turn off emails when resubmit is done. # Implmement generic way of sending data/XML ## Create application that reads in NetCDF file and generates XML to match our schema # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Cut down on JAR file bloat (happening in new arch) # Crawl all the old OASIS files and load the metadata into SSDS. # Meet with MMI to talk about pulling our SV and SU from ontologies. # XML with schemas declared not validating # Add mechanism that users can drop files of known type with all metadata in known format into a directory and SSDS will automatically register and store it. # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # Implement ingest socket -> JMS topic # Add QC flag to SSDS packets and match in database (SSDS_Data). # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # In Packet``Output``Manager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and Packet``Outputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add end of line terminator as separator in parsing packet records (not files) # Can I create something that will emulate reading from a serial port but is back by the SSDS data service? This would be done so that native applications that are used to reading from serial ports, can still do so. To be honest, this would be really cool, but seems fragile and unlikely. This started with the Nortek, so look at Nortek.no for what they called "Online applications" # Get LOBO data into SSDS # Get raw data porting from SSDS to LAS server working for Mooring data (Leverage Mike G's stuff). # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" data by storing comments with the data and returning that to user's upon request (think NetCDFs history) # Edit bad packets that are serialized to disk # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data) # Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Setup service that will calculate salinity automatically. # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Query: "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Create Process to create useable data file from inductive CTD string # Look at integrating Earth Google using services. # HOOVES Improvements: ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface # Look at NPS AUV Mission Planner as a way to interact with AUV data.Don Brutzman and Peter Flynn came to give a demo on this and it seems to be a possibility for a tool that can integrate and visualize modeling and collected data (as well as vehicle tracks). Just check Don's web address to find information ]]> 1179756 1212537 Fog-SSDS_Metadata h3. UpdateBot # Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) # Have updateBot crawl all resources and update contentLength if not specified. # Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. # Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). # Look into having SSDS create "README" type files in the same location as certain DataContainers. ## These could/should be in FGDC format(?) h3. Graphing Application (There is overlap with Mike's Post processing, we need to define logical boundaries) # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Develop web page to allow users to configure plot creation # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range h3. Transmogrify/Ingest/SQLIngest # Build non-JMS mechanism for users to send data/metadata to SSDS. # Put true exception handling in SQLIngest to throw messages when the DB can't be reached. # Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) h3. Core (EAR) # Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds # Build services to read data from DataContainers that are files through the query interface (not just from packets). # Verify (unit tests) that the RecordDescription level parse regular expression works # Finish implementing all DAOs # Make sure all methods have associated count method # Make sure all methods have boolean option for return full graph # Make sure all methods have capability to specify a sort by field # Verify returned DataContainer collections should be sorted by start date as default # Verify implemented query for DataContainer by DataContainerGroup # Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. # Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. # Write valid unit test for Object and XMLBuilders # Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. # Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) h3. Web applications # Verify PC02 plots are working after M0 turnaround # Develop admin application to edit all metadata objects and their relationships # Increase session timeout in Explorer # Add capability in Explorer to export deployment XML template from Web to help in XML authoring (or at least connect to most recent XML in CVS). ## Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Put UML diagram of data model on developer section of web app. # Implement more queries in Explorer ## Find all post products from deployment ## Find all resources of certain types (graphics, log files, calibration files, etc.) ## "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ## Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ## Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ## "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Build web pages that allow user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Finish documenting data packet structure on web pages. # Web pages to help with automated workflows(?) # Desire in long term for common 'status reporting' system for instruments (something like an ops portal) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # In Explorer, truncate long deployment names h3. Clients # Migrate HOOVES to new architecture and add improvements ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) h3. Other Project Overlaps h5. SNMP h5. ORION CI h5. CGSN h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status}]]> 1179770 1212536 Fog-SSDS_Metadata h3. UpdateBot # Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) # Have updateBot crawl all resources and update contentLength if not specified. # Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. # Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). # Look into having SSDS create "README" type files in the same location as certain DataContainers. h3. Graphing Application # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Develop web page to allow users to configure plot creation # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range h3. Transmogrify/Ingest/SQLIngest # Build non-JMS mechanism for users to send data/metadata to SSDS. # Put true exception handling in SQLIngest to throw messages when the DB can't be reached. # Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) h3. Core (EAR) # Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds # Build services to read data from DataContainers that are files through the query interface (not just from packets). # Verify (unit tests) that the RecordDescription level parse regular expression works # Finish implementing all DAOs # Make sure all methods have associated count method # Make sure all methods have boolean option for return full graph # Make sure all methods have capability to specify a sort by field # Verify returned DataContainer collections should be sorted by start date as default # Verify implemented query for DataContainer by DataContainerGroup # Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. # Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. # Write valid unit test for Object and XMLBuilders # Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. # Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) h3. Web applications # Verify PC02 plots are working after M0 turnaround # Develop admin application to edit all metadata objects and their relationships # Increase session timeout in Explorer # Add capability in Explorer to export deployment XML template from Web to help in XML authoring (or at least connect to most recent XML in CVS). ## Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Put UML diagram of data model on developer section of web app. # Implement more queries in Explorer ## Find all post products from deployment ## Find all resources of certain types (graphics, log files, calibration files, etc.) ## "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ## Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ## Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ## "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Build web pages that allow user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Finish documenting data packet structure on web pages. # Web pages to help with automated workflows(?) # Desire in long term for common 'status reporting' system for instruments (something like an ops portal) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup h3. Clients # Migrate HOOVES to new architecture and add improvements ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) h3. Other Project Overlaps h5. SNMP h5. ORION CI h5. CGSN h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status}]]> 1179769 1212533 Fog-SSDS_Metadata h3. UpdateBot # Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) # Have updateBot crawl all resources and update contentLength if not specified. # Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. # Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). # Look into having SSDS create "README" type files in the same location as certain DataContainers. h3. Graphing Application # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Develop web page to allow users to configure plot creation # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range h3. Transmogrify/Ingest/SQLIngest # Build non-JMS mechanism for users to send data/metadata to SSDS. # Put true exception handling in SQLIngest to throw messages when the DB can't be reached. # Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). h3. Core (EAR) # Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds # Build services to read data from DataContainers that are files through the query interface (not just from packets). # Verify (unit tests) that the RecordDescription level parse regular expression works # Finish implementing all DAOs # Make sure all methods have associated count method # Make sure all methods have boolean option for return full graph # Make sure all methods have capability to specify a sort by field # Verify returned DataContainer collections should be sorted by start date as default # Verify implemented query for DataContainer by DataContainerGroup # Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. # Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. # Write valid unit test for Object and XMLBuilders # Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. # Instead of using comand line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. h3. Web applications # Verify PC02 plots are working after M0 turnaround # Develop admin application to edit all metadata objects and their relationships # Increase session timeout in Explorer # Add capability in Explorer to export deployment XML template from Web to help in XML authoring (or at least connect to most recent XML in CVS). ## Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Put UML diagram of data model on developer section of web app. # Implement more queries in Explorer ## Find all post products from deployment ## Find all resources of certain types (graphics, log files, calibration files, etc.) ## "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ## Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ## Given a deployment, can I walk up the parent tree to look for a parent with a certain role? # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Build web pages that allow user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Finish documenting data packet structure on web pages. # Web pages to help with automated workflows(?) # Desire in long term for common 'status reporting' system for instruments (something like an ops portal) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. h3. Clients # Migrate HOOVES to new architecture # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) h3. Other Project Overlaps h5. SNMP h5. ORION CI h5. CGSN h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Future Tasks (Shelved) # Hold Brown Bag on architecture and uses. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional Standard``Variables, Standard``Units, Standard``Keywords, Standard``Domain, Standard``Reference``Scale, Device``Type, Resource``Type, Data``Producer``Group, Data``Container``Group and then notify the user of that change so they can change their source. # Remove Deployment info from PUCK XML (someday, probably for MSE). # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. # Write tests for Resource``BLOB->Object``Builder for byte array and verify that it is working correctly. # Send API's for SSDS ingest and Data``Stream Access to Data``Stream LOOKING group # Look at Mike Godin's Schema changes for AOSN # Can I embed the business logic documentation as Java``Doc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # See if I can use NT logins with Merlia driver in SSDS # Have the capability to turn off emails when resubmit is done. # Implmement generic way of sending data/XML ## Create application that reads in NetCDF file and generates XML to match our schema # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Cut down on JAR file bloat (happening in new arch) # Crawl all the old OASIS files and load the metadata into SSDS. # Meet with MMI to talk about pulling our SV and SU from ontologies. # XML with schemas declared not validating # Add mechanism that users can drop files of known type with all metadata in known format into a directory and SSDS will automatically register and store it. # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # Implement ingest socket -> JMS topic # Add QC flag to SSDS packets and match in database (SSDS_Data). # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # In Packet``Output``Manager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and Packet``Outputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add end of line terminator as separator in parsing packet records (not files) # Can I create something that will emulate reading from a serial port but is back by the SSDS data service? This would be done so that native applications that are used to reading from serial ports, can still do so. To be honest, this would be really cool, but seems fragile and unlikely. This started with the Nortek, so look at Nortek.no for what they called "Online applications" # Get LOBO data into SSDS # Get raw data porting from SSDS to LAS server working for Mooring data (Leverage Mike G's stuff). # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" data by storing comments with the data and returning that to user's upon request (think NetCDFs history) # Edit bad packets that are serialized to disk # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data) # Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Setup service that will calculate salinity automatically. # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Query: "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Create Process to create useable data file from inductive CTD string # Look at integrating Earth Google using services. # HOOVES Improvements: ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface # Look at NPS AUV Mission Planner as a way to interact with AUV data.Don Brutzman and Peter Flynn came to give a demo on this and it seems to be a possibility for a tool that can integrate and visualize modeling and collected data (as well as vehicle tracks). Just check Don's web address to find information ]]> 1179766 1212534 Fog-SSDS_Metadata h3. UpdateBot # Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) # Have updateBot crawl all resources and update contentLength if not specified. # Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. # Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). # Look into having SSDS create "README" type files in the same location as certain DataContainers. h3. Graphing Application # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Develop web page to allow users to configure plot creation # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range h3. Transmogrify/Ingest/SQLIngest # Build non-JMS mechanism for users to send data/metadata to SSDS. # Put true exception handling in SQLIngest to throw messages when the DB can't be reached. # Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) h3. Core (EAR) # Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds # Build services to read data from DataContainers that are files through the query interface (not just from packets). # Verify (unit tests) that the RecordDescription level parse regular expression works # Finish implementing all DAOs # Make sure all methods have associated count method # Make sure all methods have boolean option for return full graph # Make sure all methods have capability to specify a sort by field # Verify returned DataContainer collections should be sorted by start date as default # Verify implemented query for DataContainer by DataContainerGroup # Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. # Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. # Write valid unit test for Object and XMLBuilders # Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. # Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. h3. Web applications # Verify PC02 plots are working after M0 turnaround # Develop admin application to edit all metadata objects and their relationships # Increase session timeout in Explorer # Add capability in Explorer to export deployment XML template from Web to help in XML authoring (or at least connect to most recent XML in CVS). ## Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Put UML diagram of data model on developer section of web app. # Implement more queries in Explorer ## Find all post products from deployment ## Find all resources of certain types (graphics, log files, calibration files, etc.) ## "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ## Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ## Given a deployment, can I walk up the parent tree to look for a parent with a certain role? # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Build web pages that allow user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Finish documenting data packet structure on web pages. # Web pages to help with automated workflows(?) # Desire in long term for common 'status reporting' system for instruments (something like an ops portal) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup h3. Clients # Migrate HOOVES to new architecture # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) h3. Other Project Overlaps h5. SNMP h5. ORION CI h5. CGSN h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Future Tasks (Shelved) # Add QC flag to SSDS packets and match in database (SSDS_Data). # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and Packet``Outputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add end of line terminator as separator in parsing packet records (not files) # Can I create something that will emulate reading from a serial port but is back by the SSDS data service? This would be done so that native applications that are used to reading from serial ports, can still do so. To be honest, this would be really cool, but seems fragile and unlikely. This started with the Nortek, so look at Nortek.no for what they called "Online applications" # Get LOBO data into SSDS # Get raw data porting from SSDS to LAS server working for Mooring data (Leverage Mike G's stuff). # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" data by storing comments with the data and returning that to user's upon request (think NetCDFs history) # Edit bad packets that are serialized to disk # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data) # Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Setup service that will calculate salinity automatically. # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Query: "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Create Process to create useable data file from inductive CTD string # Look at integrating Earth Google using services. # HOOVES Improvements: ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface # Look at NPS AUV Mission Planner as a way to interact with AUV data.Don Brutzman and Peter Flynn came to give a demo on this and it seems to be a possibility for a tool that can integrate and visualize modeling and collected data (as well as vehicle tracks). Just check Don's web address to find information ]]> 1179767 13959216 13926449 13959214 13926447 10388142 login result page /loginResult.jsp Engineering Distribution List Research Distribution List ITD Distribution List DMO Distribution List OED Distribution List {code} # Open command prompt, cd to directory where SSDS was checked out and type {noformat}ant -Dtarget=deploy{noformat} # Using mySQL command utility, run the MySQL script to setup DB # Start JBoss # Configure mod_jk in Apache/JBoss # Configure SSL for login.jsp page Setup an Eclipse project: # Source directories should be src/java and src/gen (created by ant during build) # Add all jars in the lib directory # Add src/resources/build/antlr/antlr-2.7.5.jar # Add JBoss jars ## JBOSS_HOME/client/activation.jar ## JBOSS_HOME/client/servlet-api.jar ## JBOSS_HOME/client/jboss-j2ee.jar ## JBOSS_HOME/client/log4j.jar ## JBOSS_HOME/server/default/lib/commons-codec.jar ## JBOSS_HOME/server/default/lib/commons-collections.jar ## JBOSS_HOME/server/default/lib/commons-httpclient.jar ## JBOSS_HOME/server/default/lib/hibernate3.jar ## JBOSS_HOME/server/default/lib/mail.jar # output set to build/classes (to align with ant's build files) Create a FlexBuilder (plug-in) project: # Start Eclipse with Flex-Builder plug-in installed # File->New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/flex and choose for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). {note:title=Will create a Main.mxml} Note that the creation of the Flex Builder project will create a Main.mxml file. To fix this, right click on explorer.mxml and choose 'Set As Default Application', then you can delete the Main.mxml file. {note} NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 10355411 10388146 login result page /loginResult.jsp Engineering Distribution List Research Distribution List ITD Distribution List DMO Distribution List OED Distribution List {code} # Open command prompt, cd to directory where SSDS was checked out and type {noformat}ant -Dtarget=deploy{noformat} # Using mySQL command utility, run the MySQL script to setup DB # Start JBoss # Configure mod_jk in Apache/JBoss # Configure SSL for login.jsp page Setup an Eclipse project: # Source directories should be src/java and src/gen (created by ant during build) # Add all jars in the lib directory # Add src/resources/build/antlr/antlr-2.7.5.jar # Add JBoss jars ## JBOSS_HOME/client/activation.jar ## JBOSS_HOME/client/servlet-api.jar ## JBOSS_HOME/client/jboss-j2ee.jar ## JBOSS_HOME/client/log4j.jar ## JBOSS_HOME/server/default/lib/commons-codec.jar ## JBOSS_HOME/server/default/lib/commons-collections.jar ## JBOSS_HOME/server/default/lib/commons-httpclient.jar ## JBOSS_HOME/server/default/lib/hibernate3.jar ## JBOSS_HOME/server/default/lib/mail.jar # output set to build/classes (to align with ant's build files) Create a FlexBuilder (plug-in) project: # Start Eclipse with Flex-Builder plug-in installed # File->New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/flex and choose for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). {note:title=Will create a Main.mxml} Note that the creation of the Flex Builder project will create a Main.mxml file. To fix this, right click on explorer.mxml and choose 'Set As Default Application', then you can delete the Main.mxml file. {note} NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 10355415 10388144 login result page /loginResult.jsp Engineering Distribution List Research Distribution List ITD Distribution List DMO Distribution List OED Distribution List {code} # Open command prompt, cd to directory where SSDS was checked out and type {noformat}ant -Dtarget=deploy{noformat} # Using mySQL command utility, run the MySQL script to setup DB # Start JBoss # Configure mod_jk in Apache/JBoss # Configure SSL for login.jsp page Setup an Eclipse project: # Source directories should be src/java and src/gen (created by ant during build) # Add all jars in the lib directory # Add src/resources/build/antlr/antlr-2.7.5.jar # Add JBoss jars ## JBOSS_HOME/client/activation.jar ## JBOSS_HOME/client/servlet-api.jar ## JBOSS_HOME/client/jboss-j2ee.jar ## JBOSS_HOME/client/log4j.jar ## JBOSS_HOME/server/default/lib/commons-codec.jar ## JBOSS_HOME/server/default/lib/commons-collections.jar ## JBOSS_HOME/server/default/lib/commons-httpclient.jar ## JBOSS_HOME/server/default/lib/hibernate3.jar ## JBOSS_HOME/server/default/lib/mail.jar # output set to build/classes (to align with ant's build files) Create a FlexBuilder (plug-in) project: # Start Eclipse with Flex-Builder plug-in installed # File->New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/flex and choose for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). {note:title=Will create a Main.mxml} Note that the creation of the Flex Builder project will create a Main.mxml file. To fix this, right click on explorer.mxml and choose 'Set As Default Application', then you can delete the Main.mxml file. {note} NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 10355413 13697242 13664491 10388131 login result page /loginResult.jsp Engineering Distribution List Research Distribution List ITD Distribution List DMO Distribution List OED Distribution List {code} # Open command prompt, cd to directory where SSDS was checked out and type {noformat}ant -Dtarget=deploy{noformat} # Using mySQL command utility, run the MySQL script to setup DB # Start JBoss # Configure mod_jk in Apache/JBoss # Configure SSL for login.jsp page Setup an Eclipse project: # Source directories should be src/java and src/gen (created by ant during build) # Add all jars in the lib directory # Add src/resources/build/antlr/antlr-2.7.5.jar # Add JBoss jars ## JBOSS_HOME/client/activation.jar ## JBOSS_HOME/client/servlet-api.jar ## JBOSS_HOME/client/jboss-j2ee.jar ## JBOSS_HOME/client/log4j.jar ## JBOSS_HOME/server/default/lib/commons-codec.jar ## JBOSS_HOME/server/default/lib/commons-collections.jar ## JBOSS_HOME/server/default/lib/commons-httpclient.jar ## JBOSS_HOME/server/default/lib/hibernate3.jar ## JBOSS_HOME/server/default/lib/mail.jar # output set to build/classes (to align with ant's build files) Create a FlexBuilder (plug-in) project: # Start Eclipse with Flex-Builder plug-in installed # File->New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/flex and choose for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). {note:title=Will create a Main.mxml} Note that the creation of the Flex Builder project will create a Main.mxml file. To fix this, right click on explorer.mxml and choose 'Set As Default Application', then you can delete the Main.mxml file. {note} NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 10355400 13697240 13664489 10388137 login result page /loginResult.jsp Engineering Distribution List Research Distribution List ITD Distribution List DMO Distribution List OED Distribution List {code} # Open command prompt, cd to directory where SSDS was checked out and type {noformat}ant -Dtarget=deploy{noformat} # Using mySQL command utility, run the MySQL script to setup DB # Start JBoss # Configure mod_jk in Apache/JBoss # Configure SSL for login.jsp page Setup an Eclipse project: # Source directories should be src/java and src/gen (created by ant during build) # Add all jars in the lib directory # Add src/resources/build/antlr/antlr-2.7.5.jar # Add JBoss jars ## JBOSS_HOME/client/activation.jar ## JBOSS_HOME/client/servlet-api.jar ## JBOSS_HOME/client/jboss-j2ee.jar ## JBOSS_HOME/client/log4j.jar ## JBOSS_HOME/server/default/lib/commons-codec.jar ## JBOSS_HOME/server/default/lib/commons-collections.jar ## JBOSS_HOME/server/default/lib/commons-httpclient.jar ## JBOSS_HOME/server/default/lib/hibernate3.jar ## JBOSS_HOME/server/default/lib/mail.jar # output set to build/classes (to align with ant's build files) Create a FlexBuilder (plug-in) project: # Start Eclipse with Flex-Builder plug-in installed # File->New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/flex and choose for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). {note:title=Will create a Main.mxml} Note that the creation of the Flex Builder project will create a Main.mxml file. To fix this, right click on explorer.mxml and choose 'Set As Default Application', then you can delete the Main.mxml file. {note} NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 10355406 10388157 login result page /loginResult.jsp Engineering Distribution List Research Distribution List ITD Distribution List DMO Distribution List OED Distribution List {code} # Open command prompt, cd to directory where SSDS was checked out and type {noformat}ant -Dtarget=deploy{noformat} # Using mySQL command utility, run the MySQL script to setup DB # Start JBoss # Configure mod_jk in Apache/JBoss # Configure SSL for login.jsp page Setup an Eclipse project: # Source directories should be src/java and src/gen (created by ant during build) # Add all jars in the lib directory # Add src/resources/build/antlr/antlr-2.7.5.jar # Add JBoss jars ## JBOSS_HOME/client/activation.jar ## JBOSS_HOME/client/servlet-api.jar ## JBOSS_HOME/client/jboss-j2ee.jar ## JBOSS_HOME/client/log4j.jar ## JBOSS_HOME/server/default/lib/commons-codec.jar ## JBOSS_HOME/server/default/lib/commons-collections.jar ## JBOSS_HOME/server/default/lib/commons-httpclient.jar ## JBOSS_HOME/server/default/lib/hibernate3.jar ## JBOSS_HOME/server/default/lib/mail.jar # output set to build/classes (to align with ant's build files) Create a FlexBuilder (plug-in) project: # Start Eclipse with Flex-Builder plug-in installed # File->New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/flex and choose for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). {note:title=Will create a Main.mxml} Note that the creation of the Flex Builder project will create a Main.mxml file. To fix this, right click on explorer.mxml and choose 'Set As Default Application', then you can delete the Main.mxml file. {note} NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 10355426 10388162 login result page /loginResult.jsp Engineering Distribution List Research Distribution List ITD Distribution List DMO Distribution List OED Distribution List {noformat} # Whew, once all that is done, you are ready to build and deploy the SSDS. Open command prompt, cd to directory where SSDS was checked out and type {noformat} ant -Dtarget=deploy {noformat} # Start JBoss # Configure mod_jk in Apache/JBoss # Configure SSL for login.jsp page Setup an Eclipse project: # Source directories should be src/java and src/gen (created by ant during build) # Add all jars in the lib directory # Add src/resources/build/antlr/antlr-2.7.5.jar # Add JBoss jars ## JBOSS_HOME/client/activation.jar ## JBOSS_HOME/client/servlet-api.jar ## JBOSS_HOME/client/jboss-j2ee.jar ## JBOSS_HOME/client/log4j.jar ## JBOSS_HOME/server/default/lib/commons-codec.jar ## JBOSS_HOME/server/default/lib/commons-collections.jar ## JBOSS_HOME/server/default/lib/commons-httpclient.jar ## JBOSS_HOME/server/default/lib/hibernate3.jar ## JBOSS_HOME/server/default/lib/mail.jar # output set to build/classes (to align with ant's build files) Create a FlexBuilder (plug-in) project: # Start Eclipse with Flex-Builder plug-in installed # File->New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/flex and choose for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). {note:title=Will create a Main.mxml} Note that the creation of the Flex Builder project will create a Main.mxml file. To fix this, right click on explorer.mxml and choose 'Set As Default Application', then you can delete the Main.mxml file. {note} NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 10355431 10388160 login result page /loginResult.jsp Engineering Distribution List Research Distribution List ITD Distribution List DMO Distribution List OED Distribution List {code} # Open command prompt, cd to directory where SSDS was checked out and type {noformat}ant -Dtarget=deploy{noformat} # Using mySQL command utility, run the MySQL script to setup DB # Start JBoss # Configure mod_jk in Apache/JBoss # Configure SSL for login.jsp page Setup an Eclipse project: # Source directories should be src/java and src/gen (created by ant during build) # Add all jars in the lib directory # Add src/resources/build/antlr/antlr-2.7.5.jar # Add JBoss jars ## JBOSS_HOME/client/activation.jar ## JBOSS_HOME/client/servlet-api.jar ## JBOSS_HOME/client/jboss-j2ee.jar ## JBOSS_HOME/client/log4j.jar ## JBOSS_HOME/server/default/lib/commons-codec.jar ## JBOSS_HOME/server/default/lib/commons-collections.jar ## JBOSS_HOME/server/default/lib/commons-httpclient.jar ## JBOSS_HOME/server/default/lib/hibernate3.jar ## JBOSS_HOME/server/default/lib/mail.jar # output set to build/classes (to align with ant's build files) Create a FlexBuilder (plug-in) project: # Start Eclipse with Flex-Builder plug-in installed # File->New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/flex and choose for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). {note:title=Will create a Main.mxml} Note that the creation of the Flex Builder project will create a Main.mxml file. To fix this, right click on explorer.mxml and choose 'Set As Default Application', then you can delete the Main.mxml file. {note} NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 10355429 10388150 login result page /loginResult.jsp Engineering Distribution List Research Distribution List ITD Distribution List DMO Distribution List OED Distribution List {code} # Open command prompt, cd to directory where SSDS was checked out and type {noformat}ant -Dtarget=deploy{noformat} # Using mySQL command utility, run the MySQL script to setup DB # Start JBoss # Configure mod_jk in Apache/JBoss # Configure SSL for login.jsp page Setup an Eclipse project: # Source directories should be src/java and src/gen (created by ant during build) # Add all jars in the lib directory # Add src/resources/build/antlr/antlr-2.7.5.jar # Add JBoss jars ## JBOSS_HOME/client/activation.jar ## JBOSS_HOME/client/servlet-api.jar ## JBOSS_HOME/client/jboss-j2ee.jar ## JBOSS_HOME/client/log4j.jar ## JBOSS_HOME/server/default/lib/commons-codec.jar ## JBOSS_HOME/server/default/lib/commons-collections.jar ## JBOSS_HOME/server/default/lib/commons-httpclient.jar ## JBOSS_HOME/server/default/lib/hibernate3.jar ## JBOSS_HOME/server/default/lib/mail.jar # output set to build/classes (to align with ant's build files) Create a FlexBuilder (plug-in) project: # Start Eclipse with Flex-Builder plug-in installed # File->New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/flex and choose for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). {note:title=Will create a Main.mxml} Note that the creation of the Flex Builder project will create a Main.mxml file. To fix this, right click on explorer.mxml and choose 'Set As Default Application', then you can delete the Main.mxml file. {note} NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 10355419 10388148 login result page /loginResult.jsp Engineering Distribution List Research Distribution List ITD Distribution List DMO Distribution List OED Distribution List {code} # Open command prompt, cd to directory where SSDS was checked out and type {noformat}ant -Dtarget=deploy{noformat} # Using mySQL command utility, run the MySQL script to setup DB # Start JBoss # Configure mod_jk in Apache/JBoss # Configure SSL for login.jsp page Setup an Eclipse project: # Source directories should be src/java and src/gen (created by ant during build) # Add all jars in the lib directory # Add src/resources/build/antlr/antlr-2.7.5.jar # Add JBoss jars ## JBOSS_HOME/client/activation.jar ## JBOSS_HOME/client/servlet-api.jar ## JBOSS_HOME/client/jboss-j2ee.jar ## JBOSS_HOME/client/log4j.jar ## JBOSS_HOME/server/default/lib/commons-codec.jar ## JBOSS_HOME/server/default/lib/commons-collections.jar ## JBOSS_HOME/server/default/lib/commons-httpclient.jar ## JBOSS_HOME/server/default/lib/hibernate3.jar ## JBOSS_HOME/server/default/lib/mail.jar # output set to build/classes (to align with ant's build files) Create a FlexBuilder (plug-in) project: # Start Eclipse with Flex-Builder plug-in installed # File->New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/flex and choose for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). {note:title=Will create a Main.mxml} Note that the creation of the Flex Builder project will create a Main.mxml file. To fix this, right click on explorer.mxml and choose 'Set As Default Application', then you can delete the Main.mxml file. {note} NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 10355417 10388147 login result page /loginResult.jsp Engineering Distribution List Research Distribution List ITD Distribution List DMO Distribution List OED Distribution List {code} # Open command prompt, cd to directory where SSDS was checked out and type {noformat}ant -Dtarget=deploy{noformat} # Using mySQL command utility, run the MySQL script to setup DB # Start JBoss # Configure mod_jk in Apache/JBoss # Configure SSL for login.jsp page Setup an Eclipse project: # Source directories should be src/java and src/gen (created by ant during build) # Add all jars in the lib directory # Add src/resources/build/antlr/antlr-2.7.5.jar # Add JBoss jars ## JBOSS_HOME/client/activation.jar ## JBOSS_HOME/client/servlet-api.jar ## JBOSS_HOME/client/jboss-j2ee.jar ## JBOSS_HOME/client/log4j.jar ## JBOSS_HOME/server/default/lib/commons-codec.jar ## JBOSS_HOME/server/default/lib/commons-collections.jar ## JBOSS_HOME/server/default/lib/commons-httpclient.jar ## JBOSS_HOME/server/default/lib/hibernate3.jar ## JBOSS_HOME/server/default/lib/mail.jar # output set to build/classes (to align with ant's build files) Create a FlexBuilder (plug-in) project: # Start Eclipse with Flex-Builder plug-in installed # File->New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/flex and choose for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). {note:title=Will create a Main.mxml} Note that the creation of the Flex Builder project will create a Main.mxml file. To fix this, right click on explorer.mxml and choose 'Set As Default Application', then you can delete the Main.mxml file. {note} NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 10355416 10388154 login result page /loginResult.jsp Engineering Distribution List Research Distribution List ITD Distribution List DMO Distribution List OED Distribution List {code} # Open command prompt, cd to directory where SSDS was checked out and type {noformat}ant -Dtarget=deploy{noformat} # Using mySQL command utility, run the MySQL script to setup DB # Start JBoss # Configure mod_jk in Apache/JBoss # Configure SSL for login.jsp page Setup an Eclipse project: # Source directories should be src/java and src/gen (created by ant during build) # Add all jars in the lib directory # Add src/resources/build/antlr/antlr-2.7.5.jar # Add JBoss jars ## JBOSS_HOME/client/activation.jar ## JBOSS_HOME/client/servlet-api.jar ## JBOSS_HOME/client/jboss-j2ee.jar ## JBOSS_HOME/client/log4j.jar ## JBOSS_HOME/server/default/lib/commons-codec.jar ## JBOSS_HOME/server/default/lib/commons-collections.jar ## JBOSS_HOME/server/default/lib/commons-httpclient.jar ## JBOSS_HOME/server/default/lib/hibernate3.jar ## JBOSS_HOME/server/default/lib/mail.jar # output set to build/classes (to align with ant's build files) Create a FlexBuilder (plug-in) project: # Start Eclipse with Flex-Builder plug-in installed # File->New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/flex and choose for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). {note:title=Will create a Main.mxml} Note that the creation of the Flex Builder project will create a Main.mxml file. To fix this, right click on explorer.mxml and choose 'Set As Default Application', then you can delete the Main.mxml file. {note} NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 10355423 10388152 login result page /loginResult.jsp Engineering Distribution List Research Distribution List ITD Distribution List DMO Distribution List OED Distribution List {code} # Open command prompt, cd to directory where SSDS was checked out and type {noformat}ant -Dtarget=deploy{noformat} # Using mySQL command utility, run the MySQL script to setup DB # Start JBoss # Configure mod_jk in Apache/JBoss # Configure SSL for login.jsp page Setup an Eclipse project: # Source directories should be src/java and src/gen (created by ant during build) # Add all jars in the lib directory # Add src/resources/build/antlr/antlr-2.7.5.jar # Add JBoss jars ## JBOSS_HOME/client/activation.jar ## JBOSS_HOME/client/servlet-api.jar ## JBOSS_HOME/client/jboss-j2ee.jar ## JBOSS_HOME/client/log4j.jar ## JBOSS_HOME/server/default/lib/commons-codec.jar ## JBOSS_HOME/server/default/lib/commons-collections.jar ## JBOSS_HOME/server/default/lib/commons-httpclient.jar ## JBOSS_HOME/server/default/lib/hibernate3.jar ## JBOSS_HOME/server/default/lib/mail.jar # output set to build/classes (to align with ant's build files) Create a FlexBuilder (plug-in) project: # Start Eclipse with Flex-Builder plug-in installed # File->New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/flex and choose for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). {note:title=Will create a Main.mxml} Note that the creation of the Flex Builder project will create a Main.mxml file. To fix this, right click on explorer.mxml and choose 'Set As Default Application', then you can delete the Main.mxml file. {note} NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 10355421 13959348 13926586 13959350 13926588 1212674 Fog-SSDS_Metadata # Prepare for opening to community ## Put Copyright in all SSDS source code and zip up and make externally available. # Metadata Integrity Checking/Repairing/Enhancing ## Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## Have updateBot crawl all resources and update contentLength if not specified. ## Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. ## Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces ## Develop web page to allow administrators to configure plot creation ## Build services to read data from DataContainers that are files through the query interface (not just from packets). ## Finish implementing all DAOs ## Make sure all methods have associated count method ## Make sure all methods have boolean option for return full graph ## Make sure all methods have capability to specify a sort by field ## Verify returned DataContainer collections should be sorted by start date as default ## Verify implemented query for DataContainer by DataContainerGroup ## Look into implementing paging in services (Hibernate supports this). ## Verify PC02 plots are working after M0 turnaround ## Add links to CVS XML on device pages ## Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## In Explorer, truncate long deployment names ## Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## Migrate HOOVES to new architecture and add improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets ## Follow up on PUCK configuration tool (ACE) # Improve Data Ingest Mechanisms ## Try to change OASIS to make mooring turns less painful ## Load historical OASIS data into SSDS (data and metadata). ## Build non-JMS mechanism for users to send data/metadata to SSDS. ## Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## Change PacketSQLOutput/Input to work with any database (not just MS SQL) ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Improve Testing ## Verify (unit tests) that the RecordDescription level parse regular expression works ## Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## Write valid unit test for Object and XMLBuilders ## Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Documentation ## Put UML diagram of data model on developer section of web app. ## Finish documenting data packet structure on web pages. h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ]]> 1179909 13959343 13926581 1212673 UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Internal application Consolidation ## Look at SSDS_Metadata database for all urls that point to ssds.shore.mbari.org and see if we can point those to new-ssds or DODS. ## Look at SSDS_Metadata database for all urls that point to dods.shore.mbari.org and see if we can point those to dods.mbari.org. ## Shutdown web server on predator (dods too). ## Look at all web applications on predator and see if we simply CNAME ssds.shore.mbari.org to new-ssds, that will work OK. ## Shutdown jboss on predator. ## Plan shutdown time for predator. ## Backup files off predator. (should we migrate UpdateBot and Graphing stuff to elvis?). ## Have Pat upgrade predator to RHE. ## Reinstall updateBot and graphing software and restart. ## Figure out what ssds.mbari.org is right now, move content to web application and CNAME ssds.mbari.org to new-ssds.mbari.org (or rename new-ssds.mbari.org to ssds.mbari.org and CNAME new-ssds to ssds.mbari.org) ## Remove Microsoft SQL Server on SSDSPub ## Remove data directories on SSDPub ## Clean everything up and look at making SSDSPub just a Tomcat installation to house web applications ## Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? ## Verify data replication from tornado to ssdspub is turned off (remove data from ssdspub too) ## Remove the SSDS database from Solstice (backup first) ## Remove the SSDS database from Fog (backup first) ## Backup and remove all DTS's except on Fog for Solstice-SSDS_Metadata->Fog-SSDS_Metadata # Prepare for opening to community ## Put Copyright in all SSDS source code and zip up and make externally available. # Metadata Integrity Checking/Repairing/Enhancing ## Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## Have updateBot crawl all resources and update contentLength if not specified. ## Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. ## Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces ## Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) ## Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) ## Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). ## Develop web page to allow administrators to configure plot creation ## Make any direction plot (wind, heading, etc.) plot as points, not lines ## Put nominal lattitude and longitude in plot titles ## Have capability to turn on/off autoscale on plots and specify range ## Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds ## Build services to read data from DataContainers that are files through the query interface (not just from packets). ## Finish implementing all DAOs ## Make sure all methods have associated count method ## Make sure all methods have boolean option for return full graph ## Make sure all methods have capability to specify a sort by field ## Verify returned DataContainer collections should be sorted by start date as default ## Verify implemented query for DataContainer by DataContainerGroup ## Look into implementing paging in services (Hibernate supports this). ## Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs ## Build application to allow users to add QC flags and comments to data packets in SSDS_Data ## Verify PC02 plots are working after M0 turnaround ## Add links to CVS XML on device pages ## Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## In Explorer, truncate long deployment names ## Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## Migrate HOOVES to new architecture and add improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets ## Follow up on PUCK configuration tool (ACE) # Improve Data Ingest Mechanisms ## Try to change OASIS to make mooring turns less painful ## Load historical OASIS data into SSDS (data and metadata). ## Build non-JMS mechanism for users to send data/metadata to SSDS. ## Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## Change PacketSQLOutput/Input to work with any database (not just MS SQL) ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). ## Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet ## Add mechanism to notify users of new data arrival (method TBD) ### Look into NRSS (RSS for data streams) ## In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) ## Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # Improve Testing ## Verify (unit tests) that the RecordDescription level parse regular expression works ## Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## Write valid unit test for Object and XMLBuilders ## Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Documentation ## Put UML diagram of data model on developer section of web app. ## Finish documenting data packet structure on web pages. h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration ]]> 1179908 1212672 UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Internal application Consolidation ## Look at SSDS_Metadata database for all urls that point to ssds.shore.mbari.org and see if we can point those to new-ssds or DODS. ## Look at SSDS_Metadata database for all urls that point to dods.shore.mbari.org and see if we can point those to dods.mbari.org. ## Shutdown web server on predator (dods too). ## Look at all web applications on predator and see if we simply CNAME ssds.shore.mbari.org to new-ssds, that will work OK. ## Shutdown jboss on predator. ## Plan shutdown time for predator. ## Backup files off predator. (should we migrate UpdateBot and Graphing stuff to elvis?). ## Have Pat upgrade predator to RHE. ## Reinstall updateBot and graphing software and restart. ## Figure out what ssds.mbari.org is right now, move content to web application and CNAME ssds.mbari.org to new-ssds.mbari.org (or rename new-ssds.mbari.org to ssds.mbari.org and CNAME new-ssds to ssds.mbari.org) ## Remove Microsoft SQL Server on SSDSPub ## Remove data directories on SSDPub ## Clean everything up and look at making SSDSPub just a Tomcat installation to house web applications ## Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? ## Verify data replication from tornado to ssdspub is turned off (remove data from ssdspub too) ## Remove the SSDS database from Solstice (backup first) ## Remove the SSDS database from Fog (backup first) ## Backup and remove all DTS's except on Fog for Solstice-SSDS_Metadata->Fog-SSDS_Metadata # Prepare for opening to community ## Put Copyright in all SSDS source code and zip up and make externally available. # Metadata Integrity Checking/Repairing/Enhancing ## Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## Have updateBot crawl all resources and update contentLength if not specified. ## Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. ## Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces ## Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) ## Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) ## Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). ## Develop web page to allow administrators to configure plot creation ## Make any direction plot (wind, heading, etc.) plot as points, not lines ## Put nominal lattitude and longitude in plot titles ## Have capability to turn on/off autoscale on plots and specify range ## Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds ## Build services to read data from DataContainers that are files through the query interface (not just from packets). ## Finish implementing all DAOs ## Make sure all methods have associated count method ## Make sure all methods have boolean option for return full graph ## Make sure all methods have capability to specify a sort by field ## Verify returned DataContainer collections should be sorted by start date as default ## Verify implemented query for DataContainer by DataContainerGroup ## Look into implementing paging in services (Hibernate supports this). ## Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs ## Build application to allow users to add QC flags and comments to data packets in SSDS_Data ## Verify PC02 plots are working after M0 turnaround ## Add links to CVS XML on device pages ## Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Improve Data Ingest Mechanisms ## Build non-JMS mechanism for users to send data/metadata to SSDS. ## Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## Change PacketSQLOutput/Input to work with any database (not just MS SQL) ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). ## Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet ## Add mechanism to notify users of new data arrival (method TBD) ### Look into NRSS (RSS for data streams) ## In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) ## Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # Improve Testing ## Verify (unit tests) that the RecordDescription level parse regular expression works ## Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## Write valid unit test for Object and XMLBuilders ## Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Documentation ## Put UML diagram of data model on developer section of web app. ## Finish documenting data packet structure on web pages. # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # In Explorer, truncate long deployment names h3. Clients # Migrate HOOVES to new architecture and add improvements ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) # Try to change OASIS to make mooring turns less painful # Load historical OASIS data into SSDS (data and metadata). h3. Other Project Overlaps h5. SNMP # SSDS to track data provenance (hooks into the ESB for capturing metadata?) # Provide search tools h5. ORION CI h5. CGSN h3. Community Interest # Dalhousie # SOPAC h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) ]]> 1179907 1212671 UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Internal application Consolidation ## Look at SSDS_Metadata database for all urls that point to ssds.shore.mbari.org and see if we can point those to new-ssds or DODS. ## Look at SSDS_Metadata database for all urls that point to dods.shore.mbari.org and see if we can point those to dods.mbari.org. ## Shutdown web server on predator (dods too). ## Look at all web applications on predator and see if we simply CNAME ssds.shore.mbari.org to new-ssds, that will work OK. ## Shutdown jboss on predator. ## Plan shutdown time for predator. ## Backup files off predator. (should we migrate UpdateBot and Graphing stuff to elvis?). ## Have Pat upgrade predator to RHE. ## Reinstall updateBot and graphing software and restart. ## Figure out what ssds.mbari.org is right now, move content to web application and CNAME ssds.mbari.org to new-ssds.mbari.org (or rename new-ssds.mbari.org to ssds.mbari.org and CNAME new-ssds to ssds.mbari.org) ## Remove Microsoft SQL Server on SSDSPub ## Remove data directories on SSDPub ## Clean everything up and look at making SSDSPub just a Tomcat installation to house web applications ## Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? ## Verify data replication from tornado to ssdspub is turned off (remove data from ssdspub too) ## Remove the SSDS database from Solstice (backup first) ## Remove the SSDS database from Fog (backup first) ## Backup and remove all DTS's except on Fog for Solstice-SSDS_Metadata->Fog-SSDS_Metadata # Prepare for opening to community ## Put Copyright in all SSDS source code and zip up and make externally available. # Metadata Integrity Checking/Repairing/Enhancing ## Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## Have updateBot crawl all resources and update contentLength if not specified. ## Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. ## Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces ## Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) ## Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) ## Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). ## Develop web page to allow administrators to configure plot creation ## Make any direction plot (wind, heading, etc.) plot as points, not lines ## Put nominal lattitude and longitude in plot titles ## Have capability to turn on/off autoscale on plots and specify range ## Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds ## Build services to read data from DataContainers that are files through the query interface (not just from packets). ## Finish implementing all DAOs ## Make sure all methods have associated count method ## Make sure all methods have boolean option for return full graph ## Make sure all methods have capability to specify a sort by field ## Verify returned DataContainer collections should be sorted by start date as default ## Verify implemented query for DataContainer by DataContainerGroup ## Look into implementing paging in services (Hibernate supports this). ## Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs ## Build application to allow users to add QC flags and comments to data packets in SSDS_Data ## Verify PC02 plots are working after M0 turnaround # Improve Data Ingest Mechanisms ## Build non-JMS mechanism for users to send data/metadata to SSDS. ## Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## Change PacketSQLOutput/Input to work with any database (not just MS SQL) ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). ## Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet ## Add mechanism to notify users of new data arrival (method TBD) ### Look into NRSS (RSS for data streams) ## In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) ## Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # Improve Testing ## Verify (unit tests) that the RecordDescription level parse regular expression works ## Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## Write valid unit test for Object and XMLBuilders ## Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. h3. Web applications # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. # Increase session timeout in Explorer # Add capability in Explorer to export deployment XML template from Web to help in XML authoring (or at least connect to most recent XML in CVS). ## Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Put UML diagram of data model on developer section of web app. # Implement more queries in Explorer ## Find all post products from deployment ## Find all resources of certain types (graphics, log files, calibration files, etc.) ## "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ## Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ## Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ## "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Build web pages that allow user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Finish documenting data packet structure on web pages. # Web pages to help with automated workflows(?) # Desire in long term for common 'status reporting' system for instruments (something like an ops portal) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # In Explorer, truncate long deployment names h3. Clients # Migrate HOOVES to new architecture and add improvements ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) # Try to change OASIS to make mooring turns less painful # Load historical OASIS data into SSDS (data and metadata). h3. Other Project Overlaps h5. SNMP # SSDS to track data provenance (hooks into the ESB for capturing metadata?) # Provide search tools h5. ORION CI h5. CGSN h3. Community Interest # Dalhousie # SOPAC h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) ]]> 1179906 13697260 13664509 1212670 Fog-SSDS_Metadata # Prepare for opening to community ## Put Copyright in all SSDS source code and zip up and make externally available. # Metadata Integrity Checking/Repairing/Enhancing ## Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## Have updateBot crawl all resources and update contentLength if not specified. ## Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. ## Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) # Enhance Access Interfaces ## Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) ## Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) ## Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). ## Develop web page to allow administrators to configure plot creation ## Make any direction plot (wind, heading, etc.) plot as points, not lines ## Put nominal lattitude and longitude in plot titles ## Have capability to turn on/off autoscale on plots and specify range ## Refactor and reinstate the SQL integrity checks Rich wrote. # Improve Data Ingest Mechanisms ## Build non-JMS mechanism for users to send data/metadata to SSDS. ## Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## Change PacketSQLOutput/Input to work with any database (not just MS SQL) ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). ## Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet ## Add mechanism to notify users of new data arrival (method TBD) ### Look into NRSS (RSS for data streams) ## In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) h3. Core (EAR) # Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds # Build services to read data from DataContainers that are files through the query interface (not just from packets). # Verify (unit tests) that the RecordDescription level parse regular expression works # Finish implementing all DAOs # Make sure all methods have associated count method # Make sure all methods have boolean option for return full graph # Make sure all methods have capability to specify a sort by field # Verify returned DataContainer collections should be sorted by start date as default # Verify implemented query for DataContainer by DataContainerGroup # Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. # Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. # Write valid unit test for Object and XMLBuilders # Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. # Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Migrate Rich's SQL DB checks and automate # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) h3. Web applications # Verify PC02 plots are working after M0 turnaround # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. # Increase session timeout in Explorer # Add capability in Explorer to export deployment XML template from Web to help in XML authoring (or at least connect to most recent XML in CVS). ## Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Put UML diagram of data model on developer section of web app. # Implement more queries in Explorer ## Find all post products from deployment ## Find all resources of certain types (graphics, log files, calibration files, etc.) ## "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ## Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ## Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ## "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Build web pages that allow user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Finish documenting data packet structure on web pages. # Web pages to help with automated workflows(?) # Desire in long term for common 'status reporting' system for instruments (something like an ops portal) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # In Explorer, truncate long deployment names h3. Clients # Migrate HOOVES to new architecture and add improvements ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) # Try to change OASIS to make mooring turns less painful # Load historical OASIS data into SSDS (data and metadata). h3. Other Project Overlaps h5. SNMP # SSDS to track data provenance (hooks into the ESB for capturing metadata?) # Provide search tools h5. ORION CI h5. CGSN h3. Community Interest # Dalhousie # SOPAC h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? ]]> 1179905 18284547 18251779 1212669 Fog-SSDS_Metadata # Prepare for opening to community ## Put Copyright in all SSDS source code and zip up and make externally available. # Metadata Integrity Checking/Repairing/Enhancing ## Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## Have updateBot crawl all resources and update contentLength if not specified. ## Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. ## Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) # Enhance Access Interfaces ## Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) ## Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) ## Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). ## Develop web page to allow users to configure plot creation ## Make any direction plot (wind, heading, etc.) plot as points, not lines ## Put nominal lattitude and longitude in plot titles ## Have capability to turn on/off autoscale on plots and specify range ## Refactor and reinstate the SQL integrity checks Rich wrote. h3. Transmogrify/Ingest/SQLIngest # Build non-JMS mechanism for users to send data/metadata to SSDS. # Put true exception handling in SQLIngest to throw messages when the DB can't be reached. # Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) h3. Core (EAR) # Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds # Build services to read data from DataContainers that are files through the query interface (not just from packets). # Verify (unit tests) that the RecordDescription level parse regular expression works # Finish implementing all DAOs # Make sure all methods have associated count method # Make sure all methods have boolean option for return full graph # Make sure all methods have capability to specify a sort by field # Verify returned DataContainer collections should be sorted by start date as default # Verify implemented query for DataContainer by DataContainerGroup # Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. # Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. # Write valid unit test for Object and XMLBuilders # Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. # Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Migrate Rich's SQL DB checks and automate # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) h3. Web applications # Verify PC02 plots are working after M0 turnaround # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. # Increase session timeout in Explorer # Add capability in Explorer to export deployment XML template from Web to help in XML authoring (or at least connect to most recent XML in CVS). ## Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Put UML diagram of data model on developer section of web app. # Implement more queries in Explorer ## Find all post products from deployment ## Find all resources of certain types (graphics, log files, calibration files, etc.) ## "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ## Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ## Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ## "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Build web pages that allow user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Finish documenting data packet structure on web pages. # Web pages to help with automated workflows(?) # Desire in long term for common 'status reporting' system for instruments (something like an ops portal) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # In Explorer, truncate long deployment names h3. Clients # Migrate HOOVES to new architecture and add improvements ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) # Try to change OASIS to make mooring turns less painful # Load historical OASIS data into SSDS (data and metadata). h3. Other Project Overlaps h5. SNMP # SSDS to track data provenance (hooks into the ESB for capturing metadata?) # Provide search tools h5. ORION CI h5. CGSN h3. Community Interest # Dalhousie # SOPAC h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? ]]> 1179904 1212668 Fog-SSDS_Metadata # Prepare for opening to community ## Put Copyright in all SSDS source code and zip up and make externally available. # Metadata Integrity Checking/Repairing/Enhancing ## Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## Have updateBot crawl all resources and update contentLength if not specified. ## Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. ## Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) # Enhance Access Interfaces ## Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) ## Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) ## Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). ## Develop web page to allow users to configure plot creation ## Make any direction plot (wind, heading, etc.) plot as points, not lines ## Put nominal lattitude and longitude in plot titles ## Have capability to turn on/off autoscale on plots and specify range h3. Transmogrify/Ingest/SQLIngest # Build non-JMS mechanism for users to send data/metadata to SSDS. # Put true exception handling in SQLIngest to throw messages when the DB can't be reached. # Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) h3. Core (EAR) # Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds # Build services to read data from DataContainers that are files through the query interface (not just from packets). # Verify (unit tests) that the RecordDescription level parse regular expression works # Finish implementing all DAOs # Make sure all methods have associated count method # Make sure all methods have boolean option for return full graph # Make sure all methods have capability to specify a sort by field # Verify returned DataContainer collections should be sorted by start date as default # Verify implemented query for DataContainer by DataContainerGroup # Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. # Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. # Write valid unit test for Object and XMLBuilders # Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. # Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Migrate Rich's SQL DB checks and automate # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) h3. Web applications # Verify PC02 plots are working after M0 turnaround # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. # Increase session timeout in Explorer # Add capability in Explorer to export deployment XML template from Web to help in XML authoring (or at least connect to most recent XML in CVS). ## Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Put UML diagram of data model on developer section of web app. # Implement more queries in Explorer ## Find all post products from deployment ## Find all resources of certain types (graphics, log files, calibration files, etc.) ## "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ## Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ## Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ## "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Build web pages that allow user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Finish documenting data packet structure on web pages. # Web pages to help with automated workflows(?) # Desire in long term for common 'status reporting' system for instruments (something like an ops portal) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # In Explorer, truncate long deployment names h3. Clients # Migrate HOOVES to new architecture and add improvements ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) # Try to change OASIS to make mooring turns less painful # Load historical OASIS data into SSDS (data and metadata). h3. Other Project Overlaps h5. SNMP # SSDS to track data provenance (hooks into the ESB for capturing metadata?) # Provide search tools h5. ORION CI h5. CGSN h3. Community Interest # Dalhousie # SOPAC h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? ]]> 1179903 1212667 Fog-SSDS_Metadata # Prepare for opening to community ## Put Copyright in all SSDS source code and zip up and make externally available. # Metadata Integrity Checking/Repairing/Enhancing ## Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## Have updateBot crawl all resources and update contentLength if not specified. ## Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. ## Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) h3. Graphing Application (There is overlap with Mike's Post processing, we need to define logical boundaries) # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Develop web page to allow users to configure plot creation # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range h3. Transmogrify/Ingest/SQLIngest # Build non-JMS mechanism for users to send data/metadata to SSDS. # Put true exception handling in SQLIngest to throw messages when the DB can't be reached. # Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) h3. Core (EAR) # Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds # Build services to read data from DataContainers that are files through the query interface (not just from packets). # Verify (unit tests) that the RecordDescription level parse regular expression works # Finish implementing all DAOs # Make sure all methods have associated count method # Make sure all methods have boolean option for return full graph # Make sure all methods have capability to specify a sort by field # Verify returned DataContainer collections should be sorted by start date as default # Verify implemented query for DataContainer by DataContainerGroup # Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. # Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. # Write valid unit test for Object and XMLBuilders # Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. # Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Migrate Rich's SQL DB checks and automate # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) h3. Web applications # Verify PC02 plots are working after M0 turnaround # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. # Increase session timeout in Explorer # Add capability in Explorer to export deployment XML template from Web to help in XML authoring (or at least connect to most recent XML in CVS). ## Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Put UML diagram of data model on developer section of web app. # Implement more queries in Explorer ## Find all post products from deployment ## Find all resources of certain types (graphics, log files, calibration files, etc.) ## "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ## Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ## Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ## "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Build web pages that allow user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Finish documenting data packet structure on web pages. # Web pages to help with automated workflows(?) # Desire in long term for common 'status reporting' system for instruments (something like an ops portal) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # In Explorer, truncate long deployment names h3. Clients # Migrate HOOVES to new architecture and add improvements ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) # Try to change OASIS to make mooring turns less painful # Load historical OASIS data into SSDS (data and metadata). h3. Other Project Overlaps h5. SNMP # SSDS to track data provenance (hooks into the ESB for capturing metadata?) # Provide search tools h5. ORION CI h5. CGSN h3. Community Interest # Dalhousie # SOPAC h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? ]]> 1179902 1212666 Fog-SSDS_Metadata # Prepare for opening to community ## Put Copyright in all SSDS source code and zip up and make externally available. h3. UpdateBot # Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) # Have updateBot crawl all resources and update contentLength if not specified. # Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. # Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). # Look into having SSDS create "README" type files in the same location as certain DataContainers. ## These could/should be in FGDC format(?) h3. Graphing Application (There is overlap with Mike's Post processing, we need to define logical boundaries) # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Develop web page to allow users to configure plot creation # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range h3. Transmogrify/Ingest/SQLIngest # Build non-JMS mechanism for users to send data/metadata to SSDS. # Put true exception handling in SQLIngest to throw messages when the DB can't be reached. # Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) h3. Core (EAR) # Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds # Build services to read data from DataContainers that are files through the query interface (not just from packets). # Verify (unit tests) that the RecordDescription level parse regular expression works # Finish implementing all DAOs # Make sure all methods have associated count method # Make sure all methods have boolean option for return full graph # Make sure all methods have capability to specify a sort by field # Verify returned DataContainer collections should be sorted by start date as default # Verify implemented query for DataContainer by DataContainerGroup # Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. # Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. # Write valid unit test for Object and XMLBuilders # Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. # Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Migrate Rich's SQL DB checks and automate # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) h3. Web applications # Verify PC02 plots are working after M0 turnaround # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. # Increase session timeout in Explorer # Add capability in Explorer to export deployment XML template from Web to help in XML authoring (or at least connect to most recent XML in CVS). ## Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Put UML diagram of data model on developer section of web app. # Implement more queries in Explorer ## Find all post products from deployment ## Find all resources of certain types (graphics, log files, calibration files, etc.) ## "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ## Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ## Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ## "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Build web pages that allow user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Finish documenting data packet structure on web pages. # Web pages to help with automated workflows(?) # Desire in long term for common 'status reporting' system for instruments (something like an ops portal) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # In Explorer, truncate long deployment names h3. Clients # Migrate HOOVES to new architecture and add improvements ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) # Try to change OASIS to make mooring turns less painful # Load historical OASIS data into SSDS (data and metadata). h3. Other Project Overlaps h5. SNMP # SSDS to track data provenance (hooks into the ESB for capturing metadata?) # Provide search tools h5. ORION CI h5. CGSN h3. Community Interest # Dalhousie # SOPAC h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? ]]> 1179901 1212665 Fog-SSDS_Metadata h3. UpdateBot # Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) # Have updateBot crawl all resources and update contentLength if not specified. # Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. # Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). # Look into having SSDS create "README" type files in the same location as certain DataContainers. ## These could/should be in FGDC format(?) h3. Graphing Application (There is overlap with Mike's Post processing, we need to define logical boundaries) # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Develop web page to allow users to configure plot creation # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range h3. Transmogrify/Ingest/SQLIngest # Build non-JMS mechanism for users to send data/metadata to SSDS. # Put true exception handling in SQLIngest to throw messages when the DB can't be reached. # Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) h3. Core (EAR) # Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds # Build services to read data from DataContainers that are files through the query interface (not just from packets). # Verify (unit tests) that the RecordDescription level parse regular expression works # Finish implementing all DAOs # Make sure all methods have associated count method # Make sure all methods have boolean option for return full graph # Make sure all methods have capability to specify a sort by field # Verify returned DataContainer collections should be sorted by start date as default # Verify implemented query for DataContainer by DataContainerGroup # Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. # Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. # Write valid unit test for Object and XMLBuilders # Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. # Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Migrate Rich's SQL DB checks and automate # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) h3. Web applications # Verify PC02 plots are working after M0 turnaround # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. # Increase session timeout in Explorer # Add capability in Explorer to export deployment XML template from Web to help in XML authoring (or at least connect to most recent XML in CVS). ## Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Put UML diagram of data model on developer section of web app. # Implement more queries in Explorer ## Find all post products from deployment ## Find all resources of certain types (graphics, log files, calibration files, etc.) ## "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ## Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ## Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ## "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Build web pages that allow user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Finish documenting data packet structure on web pages. # Web pages to help with automated workflows(?) # Desire in long term for common 'status reporting' system for instruments (something like an ops portal) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # In Explorer, truncate long deployment names h3. Clients # Migrate HOOVES to new architecture and add improvements ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) # Try to change OASIS to make mooring turns less painful # Load historical OASIS data into SSDS (data and metadata). h3. Other Project Overlaps h5. SNMP # SSDS to track data provenance (hooks into the ESB for capturing metadata?) # Provide search tools h5. ORION CI h5. CGSN h3. Community Interest # Dalhousie # SOPAC h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? ]]> 1179900 1212664 Fog-SSDS_Metadata h3. UpdateBot # Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) # Have updateBot crawl all resources and update contentLength if not specified. # Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. # Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). # Look into having SSDS create "README" type files in the same location as certain DataContainers. ## These could/should be in FGDC format(?) h3. Graphing Application (There is overlap with Mike's Post processing, we need to define logical boundaries) # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Develop web page to allow users to configure plot creation # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range h3. Transmogrify/Ingest/SQLIngest # Build non-JMS mechanism for users to send data/metadata to SSDS. # Put true exception handling in SQLIngest to throw messages when the DB can't be reached. # Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) h3. Core (EAR) # Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds # Build services to read data from DataContainers that are files through the query interface (not just from packets). # Verify (unit tests) that the RecordDescription level parse regular expression works # Finish implementing all DAOs # Make sure all methods have associated count method # Make sure all methods have boolean option for return full graph # Make sure all methods have capability to specify a sort by field # Verify returned DataContainer collections should be sorted by start date as default # Verify implemented query for DataContainer by DataContainerGroup # Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. # Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. # Write valid unit test for Object and XMLBuilders # Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. # Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Migrate Rich's SQL DB checks and automate # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) h3. Web applications # Verify PC02 plots are working after M0 turnaround # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. # Increase session timeout in Explorer # Add capability in Explorer to export deployment XML template from Web to help in XML authoring (or at least connect to most recent XML in CVS). ## Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Put UML diagram of data model on developer section of web app. # Implement more queries in Explorer ## Find all post products from deployment ## Find all resources of certain types (graphics, log files, calibration files, etc.) ## "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ## Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ## Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ## "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Build web pages that allow user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Finish documenting data packet structure on web pages. # Web pages to help with automated workflows(?) # Desire in long term for common 'status reporting' system for instruments (something like an ops portal) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # In Explorer, truncate long deployment names h3. Clients # Migrate HOOVES to new architecture and add improvements ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) # Try to change OASIS to make mooring turns less painful # Load historical OASIS data into SSDS (data and metadata). h3. Other Project Overlaps h5. SNMP # SSDS to track data provenance (hooks into the ESB for capturing metadata?) # Provide search tools h5. ORION CI h5. CGSN h3. Community Interest # Dalhousie # SOPAC h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? ]]> 1179899 1212663 Fog-SSDS_Metadata h3. UpdateBot # Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) # Have updateBot crawl all resources and update contentLength if not specified. # Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. # Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). # Look into having SSDS create "README" type files in the same location as certain DataContainers. ## These could/should be in FGDC format(?) h3. Graphing Application (There is overlap with Mike's Post processing, we need to define logical boundaries) # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Develop web page to allow users to configure plot creation # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range h3. Transmogrify/Ingest/SQLIngest # Build non-JMS mechanism for users to send data/metadata to SSDS. # Put true exception handling in SQLIngest to throw messages when the DB can't be reached. # Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) h3. Core (EAR) # Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds # Build services to read data from DataContainers that are files through the query interface (not just from packets). # Verify (unit tests) that the RecordDescription level parse regular expression works # Finish implementing all DAOs # Make sure all methods have associated count method # Make sure all methods have boolean option for return full graph # Make sure all methods have capability to specify a sort by field # Verify returned DataContainer collections should be sorted by start date as default # Verify implemented query for DataContainer by DataContainerGroup # Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. # Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. # Write valid unit test for Object and XMLBuilders # Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. # Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Migrate Rich's SQL DB checks and automate # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) h3. Web applications # Verify PC02 plots are working after M0 turnaround # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. # Increase session timeout in Explorer # Add capability in Explorer to export deployment XML template from Web to help in XML authoring (or at least connect to most recent XML in CVS). ## Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Put UML diagram of data model on developer section of web app. # Implement more queries in Explorer ## Find all post products from deployment ## Find all resources of certain types (graphics, log files, calibration files, etc.) ## "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ## Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ## Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ## "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Build web pages that allow user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Finish documenting data packet structure on web pages. # Web pages to help with automated workflows(?) # Desire in long term for common 'status reporting' system for instruments (something like an ops portal) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # In Explorer, truncate long deployment names h3. Clients # Migrate HOOVES to new architecture and add improvements ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) # Try to change OASIS to make mooring turns less painful # Load historical OASIS data into SSDS (data and metadata). h3. Other Project Overlaps h5. SNMP # SSDS to track data provenance (hooks into the ESB for capturing metadata?) # Provide search tools h5. ORION CI h5. CGSN h3. Community Interest # Dalhousie # SOPAC h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? ]]> 1179898 13697252 13664501 1212662 Fog-SSDS_Metadata h3. UpdateBot # Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) # Have updateBot crawl all resources and update contentLength if not specified. # Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. # Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). # Look into having SSDS create "README" type files in the same location as certain DataContainers. ## These could/should be in FGDC format(?) h3. Graphing Application (There is overlap with Mike's Post processing, we need to define logical boundaries) # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Develop web page to allow users to configure plot creation # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range h3. Transmogrify/Ingest/SQLIngest # Build non-JMS mechanism for users to send data/metadata to SSDS. # Put true exception handling in SQLIngest to throw messages when the DB can't be reached. # Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) h3. Core (EAR) # Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds # Build services to read data from DataContainers that are files through the query interface (not just from packets). # Verify (unit tests) that the RecordDescription level parse regular expression works # Finish implementing all DAOs # Make sure all methods have associated count method # Make sure all methods have boolean option for return full graph # Make sure all methods have capability to specify a sort by field # Verify returned DataContainer collections should be sorted by start date as default # Verify implemented query for DataContainer by DataContainerGroup # Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. # Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. # Write valid unit test for Object and XMLBuilders # Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. # Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Migrate Rich's SQL DB checks and automate # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) h3. Web applications # Verify PC02 plots are working after M0 turnaround # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. # Increase session timeout in Explorer # Add capability in Explorer to export deployment XML template from Web to help in XML authoring (or at least connect to most recent XML in CVS). ## Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Put UML diagram of data model on developer section of web app. # Implement more queries in Explorer ## Find all post products from deployment ## Find all resources of certain types (graphics, log files, calibration files, etc.) ## "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ## Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ## Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ## "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Build web pages that allow user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Finish documenting data packet structure on web pages. # Web pages to help with automated workflows(?) # Desire in long term for common 'status reporting' system for instruments (something like an ops portal) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # In Explorer, truncate long deployment names h3. Clients # Migrate HOOVES to new architecture and add improvements ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) # Try to change OASIS to make mooring turns less painful # Load historical OASIS data into SSDS (data and metadata). h3. Other Project Overlaps h5. SNMP # SSDS to track data provenance (hooks into the ESB for capturing metadata?) # Provide search tools h5. ORION CI h5. CGSN h3. Community Interest # Dalhousie # SOPAC h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? ]]> 1179897 13959389 13926627 13697185 13664434 1212560 1179793 1212562 1179795 1212550 Fog-SSDS_Metadata h3. UpdateBot # Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) # Have updateBot crawl all resources and update contentLength if not specified. # Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. # Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). # Look into having SSDS create "README" type files in the same location as certain DataContainers. ## These could/should be in FGDC format(?) h3. Graphing Application (There is overlap with Mike's Post processing, we need to define logical boundaries) # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Develop web page to allow users to configure plot creation # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range h3. Transmogrify/Ingest/SQLIngest # Build non-JMS mechanism for users to send data/metadata to SSDS. # Put true exception handling in SQLIngest to throw messages when the DB can't be reached. # Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) h3. Core (EAR) # Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds # Build services to read data from DataContainers that are files through the query interface (not just from packets). # Verify (unit tests) that the RecordDescription level parse regular expression works # Finish implementing all DAOs # Make sure all methods have associated count method # Make sure all methods have boolean option for return full graph # Make sure all methods have capability to specify a sort by field # Verify returned DataContainer collections should be sorted by start date as default # Verify implemented query for DataContainer by DataContainerGroup # Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. # Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. # Write valid unit test for Object and XMLBuilders # Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. # Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) h3. Web applications # Verify PC02 plots are working after M0 turnaround # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. # Increase session timeout in Explorer # Add capability in Explorer to export deployment XML template from Web to help in XML authoring (or at least connect to most recent XML in CVS). ## Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Put UML diagram of data model on developer section of web app. # Implement more queries in Explorer ## Find all post products from deployment ## Find all resources of certain types (graphics, log files, calibration files, etc.) ## "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ## Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ## Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ## "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Build web pages that allow user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Finish documenting data packet structure on web pages. # Web pages to help with automated workflows(?) # Desire in long term for common 'status reporting' system for instruments (something like an ops portal) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # In Explorer, truncate long deployment names h3. Clients # Migrate HOOVES to new architecture and add improvements ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) h3. Other Project Overlaps h5. SNMP h5. ORION CI h5. CGSN h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status}]]> 1179783 1212551 Fog-SSDS_Metadata h3. UpdateBot # Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) # Have updateBot crawl all resources and update contentLength if not specified. # Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. # Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). # Look into having SSDS create "README" type files in the same location as certain DataContainers. ## These could/should be in FGDC format(?) h3. Graphing Application (There is overlap with Mike's Post processing, we need to define logical boundaries) # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Develop web page to allow users to configure plot creation # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range h3. Transmogrify/Ingest/SQLIngest # Build non-JMS mechanism for users to send data/metadata to SSDS. # Put true exception handling in SQLIngest to throw messages when the DB can't be reached. # Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) h3. Core (EAR) # Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds # Build services to read data from DataContainers that are files through the query interface (not just from packets). # Verify (unit tests) that the RecordDescription level parse regular expression works # Finish implementing all DAOs # Make sure all methods have associated count method # Make sure all methods have boolean option for return full graph # Make sure all methods have capability to specify a sort by field # Verify returned DataContainer collections should be sorted by start date as default # Verify implemented query for DataContainer by DataContainerGroup # Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. # Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. # Write valid unit test for Object and XMLBuilders # Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. # Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Migrate Rich's SQL DB checks and automate # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) h3. Web applications # Verify PC02 plots are working after M0 turnaround # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. # Increase session timeout in Explorer # Add capability in Explorer to export deployment XML template from Web to help in XML authoring (or at least connect to most recent XML in CVS). ## Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Put UML diagram of data model on developer section of web app. # Implement more queries in Explorer ## Find all post products from deployment ## Find all resources of certain types (graphics, log files, calibration files, etc.) ## "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ## Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ## Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ## "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Build web pages that allow user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Finish documenting data packet structure on web pages. # Web pages to help with automated workflows(?) # Desire in long term for common 'status reporting' system for instruments (something like an ops portal) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # In Explorer, truncate long deployment names h3. Clients # Migrate HOOVES to new architecture and add improvements ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) h3. Other Project Overlaps h5. SNMP h5. ORION CI h5. CGSN h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? ]]> 1179784 1212552 Fog-SSDS_Metadata h3. UpdateBot # Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) # Have updateBot crawl all resources and update contentLength if not specified. # Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. # Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). # Look into having SSDS create "README" type files in the same location as certain DataContainers. ## These could/should be in FGDC format(?) h3. Graphing Application (There is overlap with Mike's Post processing, we need to define logical boundaries) # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Develop web page to allow users to configure plot creation # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range h3. Transmogrify/Ingest/SQLIngest # Build non-JMS mechanism for users to send data/metadata to SSDS. # Put true exception handling in SQLIngest to throw messages when the DB can't be reached. # Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) h3. Core (EAR) # Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds # Build services to read data from DataContainers that are files through the query interface (not just from packets). # Verify (unit tests) that the RecordDescription level parse regular expression works # Finish implementing all DAOs # Make sure all methods have associated count method # Make sure all methods have boolean option for return full graph # Make sure all methods have capability to specify a sort by field # Verify returned DataContainer collections should be sorted by start date as default # Verify implemented query for DataContainer by DataContainerGroup # Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. # Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. # Write valid unit test for Object and XMLBuilders # Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. # Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Migrate Rich's SQL DB checks and automate # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) h3. Web applications # Verify PC02 plots are working after M0 turnaround # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. # Increase session timeout in Explorer # Add capability in Explorer to export deployment XML template from Web to help in XML authoring (or at least connect to most recent XML in CVS). ## Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Put UML diagram of data model on developer section of web app. # Implement more queries in Explorer ## Find all post products from deployment ## Find all resources of certain types (graphics, log files, calibration files, etc.) ## "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ## Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ## Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ## "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Build web pages that allow user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Finish documenting data packet structure on web pages. # Web pages to help with automated workflows(?) # Desire in long term for common 'status reporting' system for instruments (something like an ops portal) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # In Explorer, truncate long deployment names h3. Clients # Migrate HOOVES to new architecture and add improvements ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) # Try to change OASIS to make mooring turns less painful # Load historical OASIS data into SSDS (data and metadata). h3. Other Project Overlaps h5. SNMP h5. ORION CI h5. CGSN h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? ]]> 1179785 1212553 Fog-SSDS_Metadata h3. UpdateBot # Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) # Have updateBot crawl all resources and update contentLength if not specified. # Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. # Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). # Look into having SSDS create "README" type files in the same location as certain DataContainers. ## These could/should be in FGDC format(?) h3. Graphing Application (There is overlap with Mike's Post processing, we need to define logical boundaries) # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Develop web page to allow users to configure plot creation # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range h3. Transmogrify/Ingest/SQLIngest # Build non-JMS mechanism for users to send data/metadata to SSDS. # Put true exception handling in SQLIngest to throw messages when the DB can't be reached. # Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) h3. Core (EAR) # Verify the GetOriginalDataServlet on ssds.shore and ssdpsub simply forward on to the same servlet on new-ssds # Build services to read data from DataContainers that are files through the query interface (not just from packets). # Verify (unit tests) that the RecordDescription level parse regular expression works # Finish implementing all DAOs # Make sure all methods have associated count method # Make sure all methods have boolean option for return full graph # Make sure all methods have capability to specify a sort by field # Verify returned DataContainer collections should be sorted by start date as default # Verify implemented query for DataContainer by DataContainerGroup # Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. # Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. # Write valid unit test for Object and XMLBuilders # Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. # Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Look into implementing paging in services (Hibernate supports this). # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Migrate Rich's SQL DB checks and automate # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) h3. Web applications # Verify PC02 plots are working after M0 turnaround # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. # Increase session timeout in Explorer # Add capability in Explorer to export deployment XML template from Web to help in XML authoring (or at least connect to most recent XML in CVS). ## Put link on device page to get to most recent XML (link to CVS). Could this be generated from most recent deployment? # Put UML diagram of data model on developer section of web app. # Implement more queries in Explorer ## Find all post products from deployment ## Find all resources of certain types (graphics, log files, calibration files, etc.) ## "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ## Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent Data`roducer of type Deployment? ## Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ## "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Build web pages that allow user to send messages to different topics in the ingest component # Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). # Finish documenting data packet structure on web pages. # Web pages to help with automated workflows(?) # Desire in long term for common 'status reporting' system for instruments (something like an ops portal) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # In Explorer, truncate long deployment names h3. Clients # Migrate HOOVES to new architecture and add improvements ## Full edit pages for deployment information ## Tree structure for dataset variables that are functions of depth ## SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ## Faster variable list generation by using DODS rather than netCDF API ## More consistent use of resourceType contentType info (MIME types) ## Top-level data set display for platform level deployment nodes ## Additional queries: ### by standard variable name ### by lat/lon rubber band box via mini maplet gui interface ## Fix Bugs: ### Window sizing on startup ### thread/hash problem with multiple plots ### Numerics not showing for some data sets # GoogleMaps/GoogleEarth/Worldwind integration # Follow up on PUCK configuration tool (ACE) # Try to change OASIS to make mooring turns less painful # Load historical OASIS data into SSDS (data and metadata). h3. Other Project Overlaps h5. SNMP # SSDS to track data provenance (hooks into the ESB for capturing metadata?) h5. ORION CI h5. CGSN h3. Community Interest # Dalhousie # SOPAC h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? ]]> 1179786 1212571 1179804 1212576 1179809 1212575 1179808 1212578 1179811 1212577 1179810 1212568 1179801 1212570 1179803 1212569 1179802 1212589 1179822 13697211 13664460 1212590 1179823 1212587 1179820 13697213 13664462 1212588 1179821 13959423 13926661 1212593 1179826 10388171 login result page /loginResult.jsp Engineering Distribution List Research Distribution List ITD Distribution List DMO Distribution List OED Distribution List {noformat} # Whew, once all that is done, you are ready to build and deploy the SSDS. Open command prompt, cd to directory where SSDS was checked out and type {noformat} ant -Dtarget=deploy {noformat} # Once the build is complete, open a terminal window, change directory to where the JBoss home is, and start JBoss {noformat} sudo ./bin/run.sh {noformat} # Once JBoss finishes startup, you should see the line: {noformat} 17:12:41,607 INFO [Server] JBoss (MX MicroKernel) [4.2.2.GA (build: SVNTag=JBoss_4_2_2_GA date=200710221139)] Started in 19s:45ms {noformat} and hopefully you did not see any stack traces fly by during startup. If all looks OK, you should be able to go the SSDS_Metadata database and see newly created tables that will be where SSDS will store is metadata. You can also open a browser and go to [http://localhost:8080/ssds-docs/] to see a simple page with some links to SSDS documentation and various utilities and libraries. {note:title=Still to document} # Configure mod_jk in Apache/JBoss # Configure SSL for login.jsp page {note} h3. Setup an Eclipse project: In order to actually do some development on the SSDS code base, you can use an IDE like Eclipse to edit the code and use ant to deploy your changes to JBoss. To setup an Eclipse project. # Install eclipse that you download from [http://www.eclipse.org] # Startup Eclipse and select File->New->Java Project. # In the New Project Window, select 'Create project from existing source' and browse to where you checked out the SSDS source code. Once you configure the project correctly, you should have the following project settings: ## There should be two source directories: {noformat} src/java {noformat} and {noformat} src/gen {noformat} (src/gen is created by ant during build) # Add all jars in the lib directory # Add src/resources/build/antlr/antlr-2.7.5.jar # Add JBoss jars ## JBOSS_HOME/client/activation.jar ## JBOSS_HOME/client/servlet-api.jar ## JBOSS_HOME/client/jboss-j2ee.jar ## JBOSS_HOME/client/log4j.jar ## JBOSS_HOME/server/default/lib/commons-codec.jar ## JBOSS_HOME/server/default/lib/commons-collections.jar ## JBOSS_HOME/server/default/lib/commons-httpclient.jar ## JBOSS_HOME/server/default/lib/hibernate3.jar ## JBOSS_HOME/server/default/lib/mail.jar # output set to build/classes (to align with ant's build files) h3. Create a FlexBuilder (plug-in) project: # Start Eclipse with Flex-Builder plug-in installed # File->New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/flex and choose it for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). {note:title=Will create a Main.mxml} Note that the creation of the Flex Builder project will create a Main.mxml file. To fix this, right click on explorer.mxml and choose 'Set As Default Application', then you can delete the Main.mxml file. {note} NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 10355440 1212594 1179827 1212591 1179824 1212592 1179825 13697203 13664452 10388167 login result page /loginResult.jsp Engineering Distribution List Research Distribution List ITD Distribution List DMO Distribution List OED Distribution List {noformat} # Whew, once all that is done, you are ready to build and deploy the SSDS. Open command prompt, cd to directory where SSDS was checked out and type {noformat} ant -Dtarget=deploy {noformat} # Once the build is complete, open a terminal window, change directory to where the JBoss home is, and start JBoss {noformat} sudo ./bin/run.sh {noformat} # Once JBoss finishes startup, you should see the line: {noformat} 17:12:41,607 INFO [Server] JBoss (MX MicroKernel) [4.2.2.GA (build: SVNTag=JBoss_4_2_2_GA date=200710221139)] Started in 19s:45ms {noformat} and hopefully you did not see any stack traces fly by during startup. If all looks OK, you should be able to go the SSDS_Metadata database and see newly created tables that will be where SSDS will store is metadata. You can also open a browser and go to [http://localhost:8080/ssds-docs/] to see a simple page with some links to SSDS documentation and various utilities and libraries. {note:title=Still to document} # Configure mod_jk in Apache/JBoss # Configure SSL for login.jsp page {note} h3. Setup an Eclipse project: # Source directories should be src/java and src/gen (created by ant during build) # Add all jars in the lib directory # Add src/resources/build/antlr/antlr-2.7.5.jar # Add JBoss jars ## JBOSS_HOME/client/activation.jar ## JBOSS_HOME/client/servlet-api.jar ## JBOSS_HOME/client/jboss-j2ee.jar ## JBOSS_HOME/client/log4j.jar ## JBOSS_HOME/server/default/lib/commons-codec.jar ## JBOSS_HOME/server/default/lib/commons-collections.jar ## JBOSS_HOME/server/default/lib/commons-httpclient.jar ## JBOSS_HOME/server/default/lib/hibernate3.jar ## JBOSS_HOME/server/default/lib/mail.jar # output set to build/classes (to align with ant's build files) h3. Create a FlexBuilder (plug-in) project: # Start Eclipse with Flex-Builder plug-in installed # File->New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/flex and choose for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). {note:title=Will create a Main.mxml} Note that the creation of the Flex Builder project will create a Main.mxml file. To fix this, right click on explorer.mxml and choose 'Set As Default Application', then you can delete the Main.mxml file. {note} NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 10355436 13697204 13664453 10388169 login result page /loginResult.jsp Engineering Distribution List Research Distribution List ITD Distribution List DMO Distribution List OED Distribution List {noformat} # Whew, once all that is done, you are ready to build and deploy the SSDS. Open command prompt, cd to directory where SSDS was checked out and type {noformat} ant -Dtarget=deploy {noformat} # Once the build is complete, open a terminal window, change directory to where the JBoss home is, and start JBoss {noformat} sudo ./bin/run.sh {noformat} # Once JBoss finishes startup, you should see the line: {noformat} 17:12:41,607 INFO [Server] JBoss (MX MicroKernel) [4.2.2.GA (build: SVNTag=JBoss_4_2_2_GA date=200710221139)] Started in 19s:45ms {noformat} and hopefully you did not see any stack traces fly by during startup. If all looks OK, you should be able to go the SSDS_Metadata database and see newly created tables that will be where SSDS will store is metadata. You can also open a browser and go to [http://localhost:8080/ssds-docs/] to see a simple page with some links to SSDS documentation and various utilities and libraries. {note:title=Still to document} # Configure mod_jk in Apache/JBoss # Configure SSL for login.jsp page {note} h3. Setup an Eclipse project: In order to actually do some development on the SSDS code base, you can use an IDE like Eclipse to edit the code and use ant to deploy your changes to JBoss. To setup an Eclipse project. # Install eclipse that you download from [http://www.eclipse.org] # Startup Eclipse and select File->New->Java Project. # In the New Project Window, select 'Create project from existing source' and browse to where you checked out the SSDS source code. Once you configure the project correctly, you should have the following project settings: ## There should be two source directories: {noformat} src/java {noformat} and {noformat} src/gen {noformat} (src/gen is created by ant during build) # Add all jars in the lib directory # Add src/resources/build/antlr/antlr-2.7.5.jar # Add JBoss jars ## JBOSS_HOME/client/activation.jar ## JBOSS_HOME/client/servlet-api.jar ## JBOSS_HOME/client/jboss-j2ee.jar ## JBOSS_HOME/client/log4j.jar ## JBOSS_HOME/server/default/lib/commons-codec.jar ## JBOSS_HOME/server/default/lib/commons-collections.jar ## JBOSS_HOME/server/default/lib/commons-httpclient.jar ## JBOSS_HOME/server/default/lib/hibernate3.jar ## JBOSS_HOME/server/default/lib/mail.jar # output set to build/classes (to align with ant's build files) h3. Create a FlexBuilder (plug-in) project: # Start Eclipse with Flex-Builder plug-in installed # File->New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/flex and choose for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). {note:title=Will create a Main.mxml} Note that the creation of the Flex Builder project will create a Main.mxml file. To fix this, right click on explorer.mxml and choose 'Set As Default Application', then you can delete the Main.mxml file. {note} NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 10355438 8093832 {code} # It's recommended that the final XML is to be checked into the puckxml module in MBARI's CVS on moonjelly. # Submit the Metadata to SSDS using the SSDSLoads application (available at [http://new-ssds.mbari.org/ssds-docs/client/]. The \-h option provides a usage note): {code} java -jar ssdsLoads-new-ssds.jar -d 1696.xml {code} ## If a mistake is made in the metadata (e.g. forgetting the platform Deployment) it may be easier to undo the submission and start over - do a deep delete on the parent deployment if this is the case. ## Minor attribute fixes may be done by editing the database once the deployments have been loaded # Check that the metadata has been successfully loaded using the Explorer application. h2. Establish data publishing application # This step assumes that there will be some application reading data from the deployed instrument. Perhaps it is a legacy application which reads the data to load into a custom data storage or visualization system. For this application to publish data to SSDS it must have visibility of each record the instrument produces as that record needs to be "packaged" into a SensorDataPacket and "published" to SSDS. # An example Java application that will package and publish a record is below: {code} import java.io.IOException; import moos.ssds.jms.PublisherComponent; import moos.ssds.transmogrify.SSDSDevicePacket; /** *

* Publish instrument data records to the SSDS database. The client must provide * SSDS device ID, timeStamp, sequence number, and payload. *

*
* * @author : $Author: mccann $ * @version : $Revision: 1.17.2.7 $ *
*

* The * Monterey Bay Aquarium Research Institute (MBARI) provides this * documentation and code "as is", with no warranty, express * or implied, of its quality or consistency. It is provided without * support and without obligation on the part of MBARI to assist in its * use, correction, modification, or enhancement. This information * should not be published or distributed to third parties without * specific written permission from MBARI. *

*
* Copyright 2008 MBARI.
* MBARI Proprietary Information. All rights reserved.
*
*
*/ /** * @author mccann * */ public class SsdsPublisher { /** * Publish data record from an instrument to SSDS * * @param deviceID * is the SSDS Device ID for the instrument that produces the data in payload * @param epochMilliseconds * time of payload sample in milliseconds since 1/1/1970 0000 GMT * @param sequenceNumber * an incrementing number for each packet * @param payload * data from instrument */ public static void publish(long deviceID, long epochMilliseconds, long sequenceNumber, String payload) { // Create a publisher component PublisherComponent pc = new PublisherComponent(); // Create a new SensorDataPacket SSDSDevicePacket packetToSend = new SSDSDevicePacket(deviceID, payload .getBytes().length); // Set the packet type to data (0 = Metadata, 1 = Data, 2 = Message) packetToSend.setPacketType(1); // Assign the time packetToSend.setSystemTime(epochMilliseconds); // Set the parentID to 0 for a parentless deployment packetToSend.setPlatformID(0); // Set the metadataref number to 0, if the data format changes and we can // refer to a different metadata packet then this number will change. packetToSend.setMetadataRef(0); // Set the sequence number packetToSend.setSequenceNo(sequenceNumber); // Set the payload packetToSend.setDataBuffer(payload.getBytes()); // Set the record type to 1 as this is the most common situation packetToSend.setRecordType(1); try { pc.publishBytes(SSDSDevicePacket .convertToPublishableByteArray(packetToSend)); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } /** * Test of SsdsPublisher.publish() * * @param args * No arguments are taken. Test values hard coded. */ public static void main(String[] args) { /* * Example packet for EITS instrument */ long eitsDeviceID = 1696; // 1696 is actual SSDS deviceID for EITS long sampleTime = 1228431283209L; // Sample time: 2008-12-04 22:54:43 long sampleSequenceNumber = 1; // May set it to ID from DB insert // Payload values must match the RecordVariables in SSDS String samplePayload = "12.2, 4.3, 768.2, 2.3, 1.2, 0.2"; // Temp, Cond, Pres, U, V, W SsdsPublisher.publish(eitsDeviceID, sampleTime, sampleSequenceNumber, samplePayload); } } {code} # Incorporate this class into a Java application that reads the data from the instrument and calls a method like the main() example for each record # Set up the application to execute for the duration of the deployment # Configure DStoNetCDF.pl cron job execution as is done for the OASIS data processing within the ssdsadmin account on elvis. (For MARS deployments this is done for the DataProducerGroup MARS2008).]]> 8061069 13697206 13664455 1212585 1179818 8093830 {code} # It's recommended that the final XML is to be checked into the puckxml module in MBARI's CVS on moonjelly. # Submit the Metadata to SSDS using the SSDSLoads application (available at [http://new-ssds.mbari.org/ssds-docs/client/]. The \-h option provides a usage note): {code} java -jar ssdsLoads-new-ssds.jar -d 1696.xml {code} ## If a mistake is made in the metadata (e.g. forgetting the platform Deployment) it may be easier to undo the submission and start over - do a deep delete on the parent deployment if this is the case. ## Minor attribute fixes may be done by editing the database once the deployments have been loaded # Check that the metadata has been successfully loaded using the Explorer application. h2. Establish data publishing application # This step assumes that there will be some application reading data from the deployed instrument. Perhaps it is a legacy application which reads the data to load into a custom data storage or visualization system. For this application to publish data to SSDS it must have visibility of each record the instrument produces as that record needs to be "packaged" into a SensorDataPacket and "published" to SSDS. # An example Java application that will package and publish a record is below: {code} import java.io.IOException; import moos.ssds.jms.PublisherComponent; import moos.ssds.transmogrify.SSDSDevicePacket; /** *

* Publish instrument data records to the SSDS database. The client must provide * SSDS device ID, timeStamp, sequence number, and payload. *

*
* * @author : $Author: mccann $ * @version : $Revision: 1.17.2.7 $ *
*

* The * Monterey Bay Aquarium Research Institute (MBARI) provides this * documentation and code "as is", with no warranty, express * or implied, of its quality or consistency. It is provided without * support and without obligation on the part of MBARI to assist in its * use, correction, modification, or enhancement. This information * should not be published or distributed to third parties without * specific written permission from MBARI. *

*
* Copyright 2008 MBARI.
* MBARI Proprietary Information. All rights reserved.
*
*
*/ /** * @author mccann * */ public class SsdsPublisher { /** * Publish data record from an instrument to SSDS * * @param deviceID * is the SSDS Device ID for the instrument that produces the data in payload * @param epochMilliseconds * time of payload sample in milliseconds since 1/1/1970 0000 GMT * @param sequenceNumber * an incrementing number for each packet * @param payload * data from instrument */ public static void publish(long deviceID, long epochMilliseconds, long sequenceNumber, String payload) { // Create a publisher component PublisherComponent pc = new PublisherComponent(); // Create a new SensorDataPacket SSDSDevicePacket packetToSend = new SSDSDevicePacket(deviceID, payload .getBytes().length); // Set the packet type to data (0 = Metadata, 1 = Data, 2 = Message) packetToSend.setPacketType(1); // Assign the time packetToSend.setSystemTime(epochMilliseconds); // Set the parentID to 0 for a parentless deployment packetToSend.setPlatformID(0); // Set the metadataref number to 0, if the data format changes and we can // refer to a different metadata packet then this number will change. packetToSend.setMetadataRef(0); // Set the sequence number packetToSend.setSequenceNo(sequenceNumber); // Set the payload packetToSend.setDataBuffer(payload.getBytes()); // Set the record type to 1 as this is the most common situation packetToSend.setRecordType(1); try { pc.publishBytes(SSDSDevicePacket .convertToPublishableByteArray(packetToSend)); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } /** * Test of SsdsPublisher.publish() * * @param args * No arguments are taken. Test values hard coded. */ public static void main(String[] args) { /* * Example packet for EITS instrument */ long eitsDeviceID = 1696; // 1696 is actual SSDS deviceID for EITS long sampleTime = 1228431283209L; // Sample time: 2008-12-04 22:54:43 long sampleSequenceNumber = 1; // May set it to ID from DB insert // Payload values must match the RecordVariables in SSDS String samplePayload = "12.2, 4.3, 768.2, 2.3, 1.2, 0.2"; // Temp, Cond, // Pres, U, // V, W SsdsPublisher.publish(eitsDeviceID, sampleTime, sampleSequenceNumber, samplePayload); } } {code} # Incorporate this class into a Java application that reads the data from the instrument and calls a method like the main() example for each record ]]> 8061067 13697208 13664457 10388164 login result page /loginResult.jsp Engineering Distribution List Research Distribution List ITD Distribution List DMO Distribution List OED Distribution List {noformat} # Whew, once all that is done, you are ready to build and deploy the SSDS. Open command prompt, cd to directory where SSDS was checked out and type {noformat} ant -Dtarget=deploy {noformat} # Once the build is complete, open a terminal window, change directory to where the JBoss home is, and start JBoss {noformat} sudo ./bin/run.sh {noformat} {note:title=Still to document} # Configure mod_jk in Apache/JBoss # Configure SSL for login.jsp page {note} h3. Setup an Eclipse project: # Source directories should be src/java and src/gen (created by ant during build) # Add all jars in the lib directory # Add src/resources/build/antlr/antlr-2.7.5.jar # Add JBoss jars ## JBOSS_HOME/client/activation.jar ## JBOSS_HOME/client/servlet-api.jar ## JBOSS_HOME/client/jboss-j2ee.jar ## JBOSS_HOME/client/log4j.jar ## JBOSS_HOME/server/default/lib/commons-codec.jar ## JBOSS_HOME/server/default/lib/commons-collections.jar ## JBOSS_HOME/server/default/lib/commons-httpclient.jar ## JBOSS_HOME/server/default/lib/hibernate3.jar ## JBOSS_HOME/server/default/lib/mail.jar # output set to build/classes (to align with ant's build files) h3. Create a FlexBuilder (plug-in) project: # Start Eclipse with Flex-Builder plug-in installed # File->New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/flex and choose for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). {note:title=Will create a Main.mxml} Note that the creation of the Flex Builder project will create a Main.mxml file. To fix this, right click on explorer.mxml and choose 'Set As Default Application', then you can delete the Main.mxml file. {note} NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 10355433 13697209 13664458 8093828 {code} # It's recommended that the final XML is to be checked into the puckxml module in MBARI's CVS on moonjelly. # Submit the Metadata to SSDS using the SSDSLoads application (available at [http://new-ssds.mbari.org/ssds-docs/client/]. The \-h option provides a usage note): {code} java -jar ssdsLoads-new-ssds.jar -d 1696.xml {code} ## If a mistake is made in the metadata (e.g. forgetting the platform Deployment) it may be easier to undo the submission and start over - do a deep delete on the parent deployment if this is the case. ## Minor attribute fixes may be done by editing the database once the deployments have been loaded # Check that the metadata has been successfully loaded using the Explorer application. h2. Establish data publishing application # This step assumes that there will be some application reading data from the deployed instrument. Perhaps it is a legacy application which reads the data to load into a custom data storage or visualization system. For this application to publish data to SSDS it must have visibility of each record the instrument produces as that record needs to be "packaged" into a SensorDataPacket and "published" to SSDS. # An example Java application that will package and publish a record is below: # {code} import java.io.IOException; import moos.ssds.jms.PublisherComponent; import moos.ssds.transmogrify.SSDSDevicePacket; /** *

* Publish instrument data records to the SSDS database. The client must provide * SSDS device ID, timeStamp, sequence number, and payload. *

*
* * @author : $Author: mccann $ * @version : $Revision: 1.17.2.7 $ *
*

* The * Monterey Bay Aquarium Research Institute (MBARI) provides this * documentation and code "as is", with no warranty, express * or implied, of its quality or consistency. It is provided without * support and without obligation on the part of MBARI to assist in its * use, correction, modification, or enhancement. This information * should not be published or distributed to third parties without * specific written permission from MBARI. *

*
* Copyright 2008 MBARI.
* MBARI Proprietary Information. All rights reserved.
*
*
*/ /** * @author mccann * */ public class SsdsPublisher { /** * Publish data record from an instrument to SSDS * * @param deviceID * is the SSDS Device ID for the instrument that produces the data in payload * @param epochMilliseconds * time of payload sample in milliseconds since 1/1/1970 0000 GMT * @param sequenceNumber * an incrementing number for each packet * @param payload * data from instrument */ public static void publish(long deviceID, long epochMilliseconds, long sequenceNumber, String payload) { // Create a publisher component PublisherComponent pc = new PublisherComponent(); // Create a new SensorDataPacket SSDSDevicePacket packetToSend = new SSDSDevicePacket(deviceID, payload .getBytes().length); // Set the packet type to data (0 = Metadata, 1 = Data, 2 = Message) packetToSend.setPacketType(1); // Assign the time packetToSend.setSystemTime(epochMilliseconds); // Set the parentID to 0 for a parentless deployment packetToSend.setPlatformID(0); // Set the metadataref number to 0, if the data format changes and we can // refer to a different metadata packet then this number will change. packetToSend.setMetadataRef(0); // Set the sequence number packetToSend.setSequenceNo(sequenceNumber); // Set the payload packetToSend.setDataBuffer(payload.getBytes()); // Set the record type to 1 as this is the most common situation packetToSend.setRecordType(1); try { pc.publishBytes(SSDSDevicePacket .convertToPublishableByteArray(packetToSend)); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } /** * Test of SsdsPublisher.publish() * * @param args * No arguments are taken. Test values hard coded. */ public static void main(String[] args) { /* * Example packet for EITS instrument */ long eitsDeviceID = 1696; // 1696 is actual SSDS deviceID for EITS long sampleTime = 1228431283209L; // Sample time: 2008-12-04 22:54:43 long sampleSequenceNumber = 1; // May set it to ID from DB insert // Payload values must match the RecordVariables in SSDS String samplePayload = "12.2, 4.3, 768.2, 2.3, 1.2, 0.2"; // Temp, Cond, // Pres, U, // V, W SsdsPublisher.publish(eitsDeviceID, sampleTime, sampleSequenceNumber, samplePayload); } } {code}]]> 8061065 13697195 13664444 13697197 13664446 13697199 13664448 13697201 13664450 1212598 1179831 1212597 1179830 1212596 1179829 1212595 1179828 13697189 13664438 13697191 13664440 1212599 1179832 13697193 13664442 2195659 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms *(8 days - 8 KG)* ## (.5 day) Try to change OASIS to make mooring turns less painful (documentation basically) ## (5 days) Build non-JMS mechanism for users to send data/metadata to SSDS. ## (1 day) Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## (1 day) Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Improve Testing *(5 days - 3 KG, 2MM)* ## (.5 day) Verify (unit tests) that the RecordDescription level parse regular expression works ## (.5 day) Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## (.5 day) Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## (.5 day) Write valid unit test for Object and XMLBuilders ## (.5 day) Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## (.5 day) Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. ## (2 days) Improve Wrapper tests # Documentation *(6 days - 3 KG, 3 MM)* ## (.5 day) Put UML diagram of data model on developer section of web app. ## (.5 day) Finish documenting data packet structure on web pages. ## (5 days) Document Explorer, Admin app and HOOVES h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). # Load historical OASIS data into SSDS (data and metadata). *This is important but too big for this, separate project* ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). *IMPORTANT EVEN THOUGH DESCOPED* ]]> 2162893 13959445 13926683 1212739 1179975 2195657 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms *(8 days - 8 KG)* ## (.5 day) Try to change OASIS to make mooring turns less painful (documentation basically) ## (5 days) Build non-JMS mechanism for users to send data/metadata to SSDS. ## (1 day) Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## (1 day) Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Improve Testing *(5 days - 3 KG, 2MM)* ## (.5 day) Verify (unit tests) that the RecordDescription level parse regular expression works ## (.5 day) Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## (.5 day) Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## (.5 day) Write valid unit test for Object and XMLBuilders ## (.5 day) Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## (.5 day) Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. ## (2 days) Improve Wrapper tests # Documentation *(6 days - 3 KG, 3 MM)* ## (.5 day) Put UML diagram of data model on developer section of web app. ## (.5 day) Finish documenting data packet structure on web pages. ## (5 days) Document Explorer, Admin app and HOOVES h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). # Load historical OASIS data into SSDS (data and metadata). *This is important but too big for this, separate project* ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). *IMPORTANT EVEN THOUGH DESCOPED* ]]> 2162891 2195658 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms *(8 days - 8 KG)* ## (.5 day) Try to change OASIS to make mooring turns less painful (documentation basically) ## (5 days) Build non-JMS mechanism for users to send data/metadata to SSDS. ## (1 day) Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## (1 day) Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Improve Testing *(5 days - 3 KG, 2MM)* ## (.5 day) Verify (unit tests) that the RecordDescription level parse regular expression works ## (.5 day) Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## (.5 day) Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## (.5 day) Write valid unit test for Object and XMLBuilders ## (.5 day) Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## (.5 day) Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. ## (2 days) Improve Wrapper tests # Documentation *(6 days - 3 KG, 3 MM)* ## (.5 day) Put UML diagram of data model on developer section of web app. ## (.5 day) Finish documenting data packet structure on web pages. ## (5 days) Document Explorer, Admin app and HOOVES h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). # Load historical OASIS data into SSDS (data and metadata). *This is important but too big for this, separate project* ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). *IMPORTANT EVEN THOUGH DESCOPED* ]]> 2162892 2195655 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms *(8 days - 8 KG)* ## (.5 day) Try to change OASIS to make mooring turns less painful (documentation basically) ## (5 days) Build non-JMS mechanism for users to send data/metadata to SSDS. ## (1 day) Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## (1 day) Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Improve Testing *(5 days - 3 KG, 2MM)* ## (.5 day) Verify (unit tests) that the RecordDescription level parse regular expression works ## (.5 day) Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## (.5 day) Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## (.5 day) Write valid unit test for Object and XMLBuilders ## (.5 day) Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## (.5 day) Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. ## (2 days) Improve Wrapper tests # Documentation *(6 days - 3 KG, 3 MM)* ## (.5 day) Put UML diagram of data model on developer section of web app. ## (.5 day) Finish documenting data packet structure on web pages. ## (5 days) Document Explorer, Admin app and HOOVES h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). # Load historical OASIS data into SSDS (data and metadata). *This is important but too big for this, separate project* ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). *IMPORTANT EVEN THOUGH DESCOPED* ]]> 2162889 2195656 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms *(8 days - 8 KG)* ## (.5 day) Try to change OASIS to make mooring turns less painful (documentation basically) ## (5 days) Build non-JMS mechanism for users to send data/metadata to SSDS. ## (1 day) Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## (1 day) Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Improve Testing *(5 days - 3 KG, 2MM)* ## (.5 day) Verify (unit tests) that the RecordDescription level parse regular expression works ## (.5 day) Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## (.5 day) Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## (.5 day) Write valid unit test for Object and XMLBuilders ## (.5 day) Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## (.5 day) Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. ## (2 days) Improve Wrapper tests # Documentation *(6 days - 3 KG, 3 MM)* ## (.5 day) Put UML diagram of data model on developer section of web app. ## (.5 day) Finish documenting data packet structure on web pages. ## (5 days) Document Explorer, Admin app and HOOVES h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). # Load historical OASIS data into SSDS (data and metadata). *This is important but too big for this, separate project* ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). *IMPORTANT EVEN THOUGH DESCOPED* ]]> 2162890 6782979 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms *(8 days - 8 KG)* ## (.5 day) Try to change OASIS to make mooring turns less painful (documentation basically) ## (5 days) Build non-JMS mechanism for users to send data/metadata to SSDS. ## (1 day) Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## (1 day) Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Improve Testing *(5 days - 3 KG, 2MM)* ## (.5 day) Verify (unit tests) that the RecordDescription level parse regular expression works ## (.5 day) Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## (.5 day) Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## (.5 day) Write valid unit test for Object and XMLBuilders ## (.5 day) Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## (.5 day) Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. ## (2 days) Improve Wrapper tests # Documentation *(6 days - 3 KG, 3 MM)* ## (.5 day) Put UML diagram of data model on developer section of web app. ## (.5 day) Finish documenting data packet structure on web pages. ## (5 days) Document Explorer, Admin app and HOOVES h3. Tasks that were descoped from 900823 SSDS Hardening Proposal # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). # Load historical OASIS data into SSDS (data and metadata). *This is important but too big for this, separate project* ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). *IMPORTANT EVEN THOUGH DESCOPED* ]]> 6750211 6782981 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms *(8 days - 8 KG)* ## (.5 day) Try to change OASIS to make mooring turns less painful (documentation basically) ## (5 days) Build non-JMS mechanism for users to send data/metadata to SSDS. ## (1 day) Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## (1 day) Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Improve Testing *(5 days - 3 KG, 2MM)* ## (.5 day) Verify (unit tests) that the RecordDescription level parse regular expression works ## (.5 day) Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## (.5 day) Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## (.5 day) Write valid unit test for Object and XMLBuilders ## (.5 day) Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## (.5 day) Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. ## (2 days) Improve Wrapper tests # Documentation *(6 days - 3 KG, 3 MM)* ## (.5 day) Put UML diagram of data model on developer section of web app. ## (.5 day) Finish documenting data packet structure on web pages. ## (5 days) Document Explorer, Admin app and HOOVES h3. Tasks that were descoped from 900823 SSDS Hardening Proposal # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). # Load historical OASIS data into SSDS (data and metadata). *This is important but too big for this, separate project* ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). *IMPORTANT EVEN THOUGH DESCOPED* ]]> 6750213 2195693 tag. For instance any nominalLat/Lon/Depth. # If the tag has a 'name' attribute, make sure it does not have any deployment specific information in it. For example, 'ISUS Deployment' is better than 'M2 ISUS Deployment'. The reason for removing any deployment information from the XML is so that when the device moves to a different mooring, the user should not have to edit the XML. The goal is to get all the XML to a point where it never needs to be edited when an instrument is deployed (unless something in the way the data stream is generated from the instrument changes). # Go to the SSDS Device pages and verify that the all the device information (mfg, model, serial number, name, type, etc.) matches what is currently in SSDS. If any of those are different it will update the device information in SSDS when the XML comes in the data stream. # Verify RecordDescription and RecordVariables look correct. I usually go to the raw data pages in SSDS and bring up the last few packets from the device just to verify that the number of columns and bufferSeparator look about right. # Check any changes to the XML back into CVS. # Copy the XML to the \\Tornado\ssdsdata\mooring(m1\|m2)\YYYY\xml directory # Go to the \\Tornado\ssdsdata\mooring(m1\|m2)\YYYY\cfg directory. # This next step is the one that needs to be timed with the mooring turn. When the old instrument is shutdown: ## Open the ssds.cfg file in a text editor ## Find the line that shows the currently deployed instrument and copy it to a line just below it. For example, if we are replacing the GPS, it might look like this before: {panel:title=Before Copy} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} and this after: {panel:title=After Copy} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # Now change the new line to have the correct device ID and the correct XML file URL {panel:title=After Device ID update} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = GPS_TYPE3,1511,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1511.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # To clean up the previous deployment information, put start and end dates after the XML URL {panel:title=After Adding Start/End dates} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,2007/04/25 16:21:58,2007/08/01 10:00:00,TransformGPS instrument = GPS_TYPE3,1511,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1511.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # Save the cfg file. {note:title=Saving the file will make the change take hold}When the ssds.cfg file changes (saved) is when the OASIS2SSDS processing will pick up the instrument change. Now, the next time it runs it will pick up the instrument change, grab the XML file from the 'xml' directory and publish it to SSDS. It will then publish all data under the new device ID. {note} # Edit the metadata in SSDS to put a close date on the old instrument deployment in SSDS. Currently I do that using Enterprise Manager. h5. If this is being done after the fact the next steps will also need to be taken. # After the new deployment shows up in SSDS, the start time for the new deployment will need to be edited to match the actual time the instrument was swapped. # Also because the data was being published under the incorrect device ID, it will need to be moved from one database table in SSDS_Data on Solstice to another table. ## The first thing that I do is grab the timestamp from the last packet sent from the old device in the raw data page on SSDS. ## For example, I go to: [http://new-ssds.mbari.org:8080/ssds/siamRawDataStep1.jsp] and enter the old device ID and set the number of packets back to make sure it goes far enough back to cover the actual time of the instrument swap. Then click on 'Next->'. ## Once the raw data shows up, find the last packet from the old device and grab the 'SIAM Timestamp' value (not the date/time) as that will be used in the Enterprise Manager query. ## Open Enterprise Manager and navigate to the 'SSDS_Data' database on Solstice. ## Browse the tables and find the table with the device ID of the old device and right click on it and select 'Open Table->Return all rows'. ## Click on the 'SQL' button in Enterprise Manager to bring up the SQL pane. It should show the basic query which should look something like this: {noformat} SELECT * FROM [1416] {noformat} ## Now add the where clause to pick only the data that is after the timestamp you grabbed from the last packet on the web page. {note:title=Timestamps in SQL are in Seconds}A quick note here, the 'SIAM Timestamp' on the raw data page is actually in milliseconds and the database column is in seconds so you will have to remove the last three digits of the 'SIAM Timestamp' before putting it in this query. {note} {noformat} SELECT * FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ## Run this query by clicking the run button '\!' in Enterprise Manager. ## Look over the results to make sure they look about right (usually you are looking for the length of the return which should be much shorter). You can actually use a count query to see how many rows this query will return. A count query would look like: {noformat} SELECT count(*) FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ## Once you know the query is correct, copy it to the clipboard and close the query window in Enterprise Manager. ## Navigate to the table of the device you want to copy the data into and right click and select 'All Tasks->Import Data...' which will fire up the DTS wizard. ### Click on 'Next>' ### For the Data Source database choose Solstice ### Select the 'SSDS_Data' database (note you should have permissions to do all this and use your windows authentication) ### Click on 'Next>' ### The destination configuration should be already to go (Solistice and SSDS_Data database). ### Click on 'Next>' ### Select 'Use a query to specify the data to transfer' ### Click on 'Next>' ### Paste the query from your clipboard into the 'Query Statement' window (You can click on 'Parse' if you want a quick sanity check) ### Click on 'Next>' ### Click on the 'Results' entry under the 'Destination' column which will enable a drop down box. ### Choose the table of the newly installed device where you will be copying the data to. ### Click on 'Next>' ### Click on 'Next>' ### Click on 'Finish' which will copy the data. ### Once that is done, open the table of the old instrument and the SQL pane so that we can construct the delete query on the old data. ### Paste in the select query and verify it is the same data you copied over: {noformat} SELECT * FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ### If it looks good, click on the 'Change Query type ...' button in Enterprise Manager and select 'Delete'. This will change the query to a delete query. ### Run the query by click on the run '\!' button. That will remove all the data from the old instrument. That's it ... whew\! Kevin Gomes (August 3, 2007)]]> 2162927 6782982 6750214 6782986 6750218 17498275 17465509 1212678 Fog-SSDS_Metadata # Prepare for opening to community *(3 days)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(41 days - 11 KG, 10 MM, 20 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (20 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces ## Develop web page to allow administrators to configure plot creation ## Build services to read data from DataContainers that are files through the query interface (not just from packets). ## Finish implementing all DAOs ## Make sure all methods have associated count method ## Make sure all methods have boolean option for return full graph ## Make sure all methods have capability to specify a sort by field ## Verify returned DataContainer collections should be sorted by start date as default ## Verify implemented query for DataContainer by DataContainerGroup ## Look into implementing paging in services (Hibernate supports this). ## Verify PC02 plots are working after M0 turnaround ## Add links to CVS XML on device pages ## Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## In Explorer, truncate long deployment names ## Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## Migrate HOOVES to new architecture and add improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms ## Try to change OASIS to make mooring turns less painful ## Load historical OASIS data into SSDS (data and metadata). ## Build non-JMS mechanism for users to send data/metadata to SSDS. ## Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## Change PacketSQLOutput/Input to work with any database (not just MS SQL) ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Improve Testing ## Verify (unit tests) that the RecordDescription level parse regular expression works ## Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## Write valid unit test for Object and XMLBuilders ## Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Documentation ## Put UML diagram of data model on developer section of web app. ## Finish documenting data packet structure on web pages. h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) ]]> 1179913 2195601 2162835 1212677 Fog-SSDS_Metadata # Prepare for opening to community *(3 days)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing ## Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## Have updateBot crawl all resources and update contentLength if not specified. ## Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. ## Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces ## Develop web page to allow administrators to configure plot creation ## Build services to read data from DataContainers that are files through the query interface (not just from packets). ## Finish implementing all DAOs ## Make sure all methods have associated count method ## Make sure all methods have boolean option for return full graph ## Make sure all methods have capability to specify a sort by field ## Verify returned DataContainer collections should be sorted by start date as default ## Verify implemented query for DataContainer by DataContainerGroup ## Look into implementing paging in services (Hibernate supports this). ## Verify PC02 plots are working after M0 turnaround ## Add links to CVS XML on device pages ## Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## In Explorer, truncate long deployment names ## Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## Migrate HOOVES to new architecture and add improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms ## Try to change OASIS to make mooring turns less painful ## Load historical OASIS data into SSDS (data and metadata). ## Build non-JMS mechanism for users to send data/metadata to SSDS. ## Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## Change PacketSQLOutput/Input to work with any database (not just MS SQL) ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Improve Testing ## Verify (unit tests) that the RecordDescription level parse regular expression works ## Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## Write valid unit test for Object and XMLBuilders ## Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Documentation ## Put UML diagram of data model on developer section of web app. ## Finish documenting data packet structure on web pages. h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) ]]> 1179912 2195602 2162836 1212676 Fog-SSDS_Metadata # Prepare for opening to community ## Put Copyright in all SSDS source code and zip up and make externally available. # Metadata Integrity Checking/Repairing/Enhancing ## Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## Have updateBot crawl all resources and update contentLength if not specified. ## Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. ## Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces ## Develop web page to allow administrators to configure plot creation ## Build services to read data from DataContainers that are files through the query interface (not just from packets). ## Finish implementing all DAOs ## Make sure all methods have associated count method ## Make sure all methods have boolean option for return full graph ## Make sure all methods have capability to specify a sort by field ## Verify returned DataContainer collections should be sorted by start date as default ## Verify implemented query for DataContainer by DataContainerGroup ## Look into implementing paging in services (Hibernate supports this). ## Verify PC02 plots are working after M0 turnaround ## Add links to CVS XML on device pages ## Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## In Explorer, truncate long deployment names ## Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## Migrate HOOVES to new architecture and add improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets ## Follow up on PUCK configuration tool (ACE) # Improve Data Ingest Mechanisms ## Try to change OASIS to make mooring turns less painful ## Load historical OASIS data into SSDS (data and metadata). ## Build non-JMS mechanism for users to send data/metadata to SSDS. ## Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## Change PacketSQLOutput/Input to work with any database (not just MS SQL) ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Improve Testing ## Verify (unit tests) that the RecordDescription level parse regular expression works ## Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## Write valid unit test for Object and XMLBuilders ## Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Documentation ## Put UML diagram of data model on developer section of web app. ## Finish documenting data packet structure on web pages. h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? ]]> 1179911 1212675 Fog-SSDS_Metadata # Prepare for opening to community ## Put Copyright in all SSDS source code and zip up and make externally available. # Metadata Integrity Checking/Repairing/Enhancing ## Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## Have updateBot crawl all resources and update contentLength if not specified. ## Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without. ## Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces ## Develop web page to allow administrators to configure plot creation ## Build services to read data from DataContainers that are files through the query interface (not just from packets). ## Finish implementing all DAOs ## Make sure all methods have associated count method ## Make sure all methods have boolean option for return full graph ## Make sure all methods have capability to specify a sort by field ## Verify returned DataContainer collections should be sorted by start date as default ## Verify implemented query for DataContainer by DataContainerGroup ## Look into implementing paging in services (Hibernate supports this). ## Verify PC02 plots are working after M0 turnaround ## Add links to CVS XML on device pages ## Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## In Explorer, truncate long deployment names ## Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## Migrate HOOVES to new architecture and add improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets ## Follow up on PUCK configuration tool (ACE) # Improve Data Ingest Mechanisms ## Try to change OASIS to make mooring turns less painful ## Load historical OASIS data into SSDS (data and metadata). ## Build non-JMS mechanism for users to send data/metadata to SSDS. ## Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## Change PacketSQLOutput/Input to work with any database (not just MS SQL) ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Improve Testing ## Verify (unit tests) that the RecordDescription level parse regular expression works ## Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## Write valid unit test for Object and XMLBuilders ## Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Documentation ## Put UML diagram of data model on developer section of web app. ## Finish documenting data packet structure on web pages. h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ]]> 1179910 1212682 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms ## Try to change OASIS to make mooring turns less painful ## Load historical OASIS data into SSDS (data and metadata). ## Build non-JMS mechanism for users to send data/metadata to SSDS. ## Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## Change PacketSQLOutput/Input to work with any database (not just MS SQL) ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Improve Testing ## Verify (unit tests) that the RecordDescription level parse regular expression works ## Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## Write valid unit test for Object and XMLBuilders ## Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Documentation ## Put UML diagram of data model on developer section of web app. ## Finish documenting data packet structure on web pages. h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). ]]> 1179917 1212681 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## Verify PC02 plots are working after M0 turnaround ## Add links to CVS XML on device pages ## Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## In Explorer, truncate long deployment names ## Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## Migrate HOOVES to new architecture and add improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms ## Try to change OASIS to make mooring turns less painful ## Load historical OASIS data into SSDS (data and metadata). ## Build non-JMS mechanism for users to send data/metadata to SSDS. ## Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## Change PacketSQLOutput/Input to work with any database (not just MS SQL) ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Improve Testing ## Verify (unit tests) that the RecordDescription level parse regular expression works ## Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## Write valid unit test for Object and XMLBuilders ## Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Documentation ## Put UML diagram of data model on developer section of web app. ## Finish documenting data packet structure on web pages. h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). ]]> 1179916 1212680 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(41 days - 11 KG, 10 MM, 20 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (20 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces ## (2 days) Develop web page to allow administrators to configure plot creation ## Finish implementing all DAOs ## Make sure all methods have associated count method ## Make sure all methods have boolean option for return full graph ## Make sure all methods have capability to specify a sort by field ## Verify returned DataContainer collections should be sorted by start date as default ## Verify implemented query for DataContainer by DataContainerGroup ## Look into implementing paging in services (Hibernate supports this). ## Verify PC02 plots are working after M0 turnaround ## Add links to CVS XML on device pages ## Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## In Explorer, truncate long deployment names ## Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## Migrate HOOVES to new architecture and add improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms ## Try to change OASIS to make mooring turns less painful ## Load historical OASIS data into SSDS (data and metadata). ## Build non-JMS mechanism for users to send data/metadata to SSDS. ## Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## Change PacketSQLOutput/Input to work with any database (not just MS SQL) ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Improve Testing ## Verify (unit tests) that the RecordDescription level parse regular expression works ## Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## Write valid unit test for Object and XMLBuilders ## Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Documentation ## Put UML diagram of data model on developer section of web app. ## Finish documenting data packet structure on web pages. h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) ]]> 1179915 1212679 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(41 days - 11 KG, 10 MM, 20 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (20 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces ## Develop web page to allow administrators to configure plot creation ## Build services to read data from DataContainers that are files through the query interface (not just from packets). ## Finish implementing all DAOs ## Make sure all methods have associated count method ## Make sure all methods have boolean option for return full graph ## Make sure all methods have capability to specify a sort by field ## Verify returned DataContainer collections should be sorted by start date as default ## Verify implemented query for DataContainer by DataContainerGroup ## Look into implementing paging in services (Hibernate supports this). ## Verify PC02 plots are working after M0 turnaround ## Add links to CVS XML on device pages ## Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## In Explorer, truncate long deployment names ## Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## Migrate HOOVES to new architecture and add improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms ## Try to change OASIS to make mooring turns less painful ## Load historical OASIS data into SSDS (data and metadata). ## Build non-JMS mechanism for users to send data/metadata to SSDS. ## Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## Change PacketSQLOutput/Input to work with any database (not just MS SQL) ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Improve Testing ## Verify (unit tests) that the RecordDescription level parse regular expression works ## Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## Write valid unit test for Object and XMLBuilders ## Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Documentation ## Put UML diagram of data model on developer section of web app. ## Finish documenting data packet structure on web pages. h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) ]]> 1179914 1212686 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms *(8 days - 8 KG)* ## (.5 day) Try to change OASIS to make mooring turns less painful (documentation basically) ## (5 days) Build non-JMS mechanism for users to send data/metadata to SSDS. ## (1 day) Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## (1 day) Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Improve Testing *(3 days - 3 KG)* ## (.5 day) Verify (unit tests) that the RecordDescription level parse regular expression works ## (.5 day) Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## (.5 day) Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## (.5 day) Write valid unit test for Object and XMLBuilders ## (.5 day) Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## (.5 day) Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Documentation *(6 days - 3 KG, 3 MM)* ## (.5 day) Put UML diagram of data model on developer section of web app. ## (.5 day) Finish documenting data packet structure on web pages. ## (5 days) Document Explorer, Admin app and HOOVES h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). # Load historical OASIS data into SSDS (data and metadata). *This is important but too big for this, separate project* ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). *IMPORTANT EVEN THOUGH DESCOPED* ]]> 1179921 1212685 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms *(8 days - 8 KG)* ## (.5 day) Try to change OASIS to make mooring turns less painful (documentation basically) ## (5 days) Build non-JMS mechanism for users to send data/metadata to SSDS. ## (1 day) Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## (1 day) Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Improve Testing *(3 days - 3 KG)* ## (.5 day) Verify (unit tests) that the RecordDescription level parse regular expression works ## (.5 day) Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## (.5 day) Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## (.5 day) Write valid unit test for Object and XMLBuilders ## (.5 day) Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## (.5 day) Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Documentation *(6 days - 3 KG, 3 MM)* ## (.5 day) Put UML diagram of data model on developer section of web app. ## (.5 day) Finish documenting data packet structure on web pages. ## (5 days) Document Explorer, Admin app and HOOVES h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). # Load historical OASIS data into SSDS (data and metadata). *This is important but too big for this, separate project* ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). *IMPORTANT EVEN THOUGH DESCOPED* ]]> 1179920 1212684 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms *(8 days - 8 KG)* ## (.5 day) Try to change OASIS to make mooring turns less painful (documentation basically) ## (5 days) Build non-JMS mechanism for users to send data/metadata to SSDS. ## (1 day) Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## (1 day) Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Improve Testing *(3 days)* ## (.5 day) Verify (unit tests) that the RecordDescription level parse regular expression works ## (.5 day) Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## (.5 day) Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## (.5 day) Write valid unit test for Object and XMLBuilders ## (.5 day) Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## (.5 day) Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Documentation *(6 days)* ## (.5 day) Put UML diagram of data model on developer section of web app. ## (.5 day) Finish documenting data packet structure on web pages. ## (5 days) Document Explorer, Admin app and HOOVES h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). # Load historical OASIS data into SSDS (data and metadata). *This is important but too big for this, separate project* ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). *IMPORTANT EVEN THOUGH DESCOPED* ]]> 1179919 1212683 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms ## Try to change OASIS to make mooring turns less painful ## Build non-JMS mechanism for users to send data/metadata to SSDS. ## Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## Change PacketSQLOutput/Input to work with any database (not just MS SQL) ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). # Improve Testing ## Verify (unit tests) that the RecordDescription level parse regular expression works ## Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## Write valid unit test for Object and XMLBuilders ## Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. # Documentation ## Put UML diagram of data model on developer section of web app. ## Finish documenting data packet structure on web pages. h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). # Load historical OASIS data into SSDS (data and metadata). *This is important but too big for this, separate project* ]]> 1179918 1212687 1179922 1212723 1179958 1212724 1179959 1212725 1179960 1212726 1179961 9831144 9798394 1212738 1179974 17498270 17465504 1212709 1179944 17498273 17465507 1212712 tag. For instance any nominalLat/Lon/Depth. # If the tag has a 'name' attribute, make sure it does not have any deployment specific information in it. For example, 'ISUS Deployment' is better than 'M2 ISUS Deployment'. The reason for removing any deployment information from the XML is so that when the device moves to a different mooring, the user's should not have to edit the XML. The goal is to get all the XML to a point where it never needs to be edited when an instrument is deployed (unless something in the way the data stream is generated from the instrument changes). # Go to the SSDS Device pages and verify that the all the device information (mfg, model, serial number, name, type, etc.) matches what is currently in SSDS. If any of those are different it will update the device information in SSDS when the XML comes in the data stream. # Verify RecordDescription and RecordVariables look correct. I usually go to the raw data pages in SSDS and bring up the last few packets from the device just to verify that the number of columns and bufferSeparator look about right. # Check any changes to the XML back into CVS. # Copy the XML to the \\Tornado\ssdsdata\mooring\(m1|m2)\YYYY\xml directory # Go to the \\Tornado\ssdsdata\mooring\(m1|m2)\YYYY\cfg directory. # This next steps is the one that needs to be timed with the mooring turn. When the old instrument is shutdown: ## Open the ssds.cfg file in a text editor ## Find the line that show the currently deployed instrument and copy it to a line just below it. For example, if we are replacing the GPS, it might look like this before: {panel:title=Before Copy} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} and this after: {panel:title=After Copy} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # Now change the new line to have the correct device ID and the correct XML file URL {panel:title=After Device ID update} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = GPS_TYPE3,1511,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1511.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # To clean up the previous deployment information, put start and end dates after the XML URL {panel:title=After Adding Start/End dates} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,2007/04/25 16:21:58,2007/08/01 10:00:00,TransformGPS instrument = GPS_TYPE3,1511,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1511.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # Save the cfg file. {note:title=Saving the file will make the change take hold} When the ssds.cfg file changes (saved) is when the OASIS2SSDS processing will pick up the instrument change. Now, the next time it runs it will pick up the instrument change, grab the XML file from the 'xml' directory and publish it to SSDS. It will then publish all data under the new device ID. {note} # Edit the metadata in SSDS to put a close date on the old instrument deployment in SSDS. Currently I do that using Enterprise Manager. h5. If this is being done after the fact the next steps will also need to be taken. # After the new deployment shows up in SSDS, the start time for the new deployment will need to be edited to match the actual time the instrument was swapped. # Also because the data was being published under the incorrect device ID, it will need to be moved from one database table in SSDS_Data on Solstice to another table. ## The first thing that I do is grab the timestamp from the last packet sent from the old device in the raw data page on SSDS. ## For example, I go to: http://new-ssds.mbari.org:8080/ssds/siamRawDataStep1.jsp and enter the old device ID and set the number of packets back to make sure it goes far enough back to cover the actual time of the instrument swap. Then click on 'Next->'. ## Once the raw data shows up, find the last packet from the old device and grab the 'SIAM Timestamp' value (not the date/time) as that will be used in the Enterprise Manager query. ## Open Enterprise Manager and navigate to the 'SSDS_Data' database on Solstice. ## Browse the tables and find the table with the device ID of the old device and right click on it and select 'Open Table->Return all rows'. ## Click on the 'SQL' button in Enterprise Manager to bring up the SQL pane. It should show the basic query which should look something like this: {noformat} SELECT * FROM [1416] {noformat} ## Now add the where clause to pick only the data that is after the timestamp you grabbed from the last packet on the web page. {note:title=Timestamps in SQL are in Seconds} A quick note here, the 'SIAM Timestamp' on the raw data page is actually in milliseconds and the database column is in seconds so you will have to remove the last three digits of the 'SIAM Timestamp' before putting it in this query. {note} {noformat} SELECT * FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ## Run this query by clicking the run button '!' in Enterprise Manager. ## Look over the results to make sure they look about right (usually you are looking for the length of the return which should be much shorter). You can actually use a count query to see how many rows this query will return. A count query would look like: {noformat} SELECT count(*) FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ## Once you know the query is correct, copy it to the clipboard and close the query window in Enterprise Manager. ## Navigate to the table of the device you want to copy the data into and right click and select 'All Tasks->Import Data...' which will fire up the DTS wizard. ### Click on 'Next>' ### For the Data Source database choose Solstice ### Select the 'SSDS_Data' database (note you should have permissions to do all this and use your windows authentication) ### Click on 'Next>' ### The destination configuration should be already to go (Solistice and SSDS_Data database). ### Click on 'Next>' ### Select 'Use a query to specify the data to transfer' ### Click on 'Next>' ### Paste the query from your clipboard into the 'Query Statement' window (You can click on 'Parse' if you want a quick sanity check) ### Click on 'Next>' ### Click on the 'Results' entry under the 'Destination' column which will enable a drop down box. ### Choose the table of the newly installed device where you will be copying the data to. ### Click on 'Next>' ### Click on 'Next>' ### Click on 'Finish' which will copy the data. ### Once that is done, open the table of the old instrument and the SQL pane so that we can construct the delete query on the old data. ### Paste in the select query and verify it is the same data you copied over: {noformat} SELECT * FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ### If it looks good, click on the 'Change Query type ...' button in Enterprise Manager and select 'Delete'. This will change the query to a delete query. ### Run the query by click on the run '!' button. That will remove all the data from the old instrument. That's it ... whew! Kevin Gomes (August 3, 2007)]]> 1179947 1212711 tag. For instance any nominalLat/Lon/Depth. # If the tag has a 'name' attribute, make sure it does not have any deployment specific information in it. For example, 'ISUS Deployment' is better than 'M2 ISUS Deployment'. The reason for removing any deployment information from the XML is so that when the device moves to a different mooring, the user's should not have to edit the XML. The goal is to get all the XML to a point where it never needs to be edited when an instrument is deployed (unless something in the way the data stream is generated from the instrument changes). # Go to the SSDS Device pages and verify that the all the device information (mfg, model, serial number, name, type, etc.) matches what is currently in SSDS. If any of those are different it will update the device information in SSDS when the XML comes in the data stream. # Verify RecordDescription and RecordVariables look correct. I usually go to the raw data pages in SSDS and bring up the last few packets from the device just to verify that the number of columns and bufferSeparator look about right. # Check any changes to the XML back into CVS. # Copy the XML to the \\Tornado\ssdsdata\mooring\(m1|m2)\YYYY\xml directory # Go to the \\Tornado\ssdsdata\mooring\(m1|m2)\YYYY\cfg directory. # This next steps is the one that needs to be timed with the mooring turn. When the old instrument is shutdown: ## Open the ssds.cfg file in a text editor ## Find the line that show the currently deployed instrument and copy it to a line just below it. For example, if we are replacing the GPS, it might look like this before: {panel:title=Before Copy} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} and this after: {panel:title=After Copy} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # Now change the new line to have the correct device ID and the correct XML file URL {panel:title=After Device ID update} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = GPS_TYPE3,1511,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1511.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # To clean up the previous deployment information, put start and end dates after the XML URL {panel:title=After Adding Start/End dates} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,2007/04/25 16:21:58,2007/08/01 10:00:00,TransformGPS instrument = GPS_TYPE3,1511,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1511.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # Save the cfg file. {note:title=Saving the file will make the change take hold} When the ssds.cfg file changes (saved) is when the OASIS2SSDS processing will pick up the instrument change. Now, the next time it runs it will pick up the instrument change, grab the XML file from the 'xml' directory and publish it to SSDS. It will then publish all data under the new device ID. {note} # Edit the metadata in SSDS to put a close date on the old instrument deployment in SSDS. Currently I do that using Enterprise Manager. h5. If this is being done after the fact the next steps will also need to be taken. # After the new deployment shows up in SSDS, the start time for the new deployment will need to be edited to match the actual time the instrument was swapped. # Also because the data was being published under the incorrect device ID, it will need to be moved from one database table in SSDS_Data on Solstice to another table.]]> 1179946 17498272 17465506 1212716 1179951 1212718 1179953 17498265 17465499 1212720 1179955 4489257 4456490 10388412 10355684 4489258 4456491 13172817 13140051 4489256 4456489 4489253 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms *(8 days - 8 KG)* ## (.5 day) Try to change OASIS to make mooring turns less painful (documentation basically) ## (5 days) Build non-JMS mechanism for users to send data/metadata to SSDS. ## (1 day) Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## (1 day) Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Improve Testing *(5 days - 3 KG, 2MM)* ## (.5 day) Verify (unit tests) that the RecordDescription level parse regular expression works ## (.5 day) Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## (.5 day) Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## (.5 day) Write valid unit test for Object and XMLBuilders ## (.5 day) Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## (.5 day) Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. ## (2 days) Improve Wrapper tests # Documentation *(6 days - 3 KG, 3 MM)* ## (.5 day) Put UML diagram of data model on developer section of web app. ## (.5 day) Finish documenting data packet structure on web pages. ## (5 days) Document Explorer, Admin app and HOOVES h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). # Load historical OASIS data into SSDS (data and metadata). *This is important but too big for this, separate project* ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). *IMPORTANT EVEN THOUGH DESCOPED* ]]> 4456486 13172811 Administrative Tools->Internet Information Services, right-click on Default Web Site and select Properties. ## Then on the Home Directory tab, check the box next to Directory browsing. # I created the directory C:\Inetpub\wwwroot\ssdsdata where I will have SSDS store everything. With the data there, it will then be HTTP accessible. # I then browsed to http://jboss.org and downloaded JBoss 4.2.2GA and unzipped it to the C:\bin\jboss-4.2.2GA directory ## I started up JBoss from the command line and noticed that port 8080 had been taken by the SQL services. I shutdown all SQL services, started JBoss so it could claim those ports, then restarted the SQL components. # I then installed SmartSVN so that I could get the code from the Google SVN server. # I started up SmartSVN and added a repository for Google code: ## SVN Location: http://shore-side-data-system.googlecode.com/svn/trunk ## Read only username: shore-side-data-system-read-only # I checked the code out read-only to a directory with the name of the server (amphisbaena.usc.edu). ]]> 13140045 4489254 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms *(8 days - 8 KG)* ## (.5 day) Try to change OASIS to make mooring turns less painful (documentation basically) ## (5 days) Build non-JMS mechanism for users to send data/metadata to SSDS. ## (1 day) Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## (1 day) Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Improve Testing *(5 days - 3 KG, 2MM)* ## (.5 day) Verify (unit tests) that the RecordDescription level parse regular expression works ## (.5 day) Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## (.5 day) Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## (.5 day) Write valid unit test for Object and XMLBuilders ## (.5 day) Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## (.5 day) Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. ## (2 days) Improve Wrapper tests # Documentation *(6 days - 3 KG, 3 MM)* ## (.5 day) Put UML diagram of data model on developer section of web app. ## (.5 day) Finish documenting data packet structure on web pages. ## (5 days) Document Explorer, Admin app and HOOVES h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). # Load historical OASIS data into SSDS (data and metadata). *This is important but too big for this, separate project* ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). *IMPORTANT EVEN THOUGH DESCOPED* ]]> 4456487 10388415 10355687 4489251 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms *(8 days - 8 KG)* ## (.5 day) Try to change OASIS to make mooring turns less painful (documentation basically) ## (5 days) Build non-JMS mechanism for users to send data/metadata to SSDS. ## (1 day) Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## (1 day) Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Improve Testing *(5 days - 3 KG, 2MM)* ## (.5 day) Verify (unit tests) that the RecordDescription level parse regular expression works ## (.5 day) Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## (.5 day) Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## (.5 day) Write valid unit test for Object and XMLBuilders ## (.5 day) Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## (.5 day) Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. ## (2 days) Improve Wrapper tests # Documentation *(6 days - 3 KG, 3 MM)* ## (.5 day) Put UML diagram of data model on developer section of web app. ## (.5 day) Finish documenting data packet structure on web pages. ## (5 days) Document Explorer, Admin app and HOOVES h3. Bugs and assigned tasks {jiraissues:http://oceana.shore.mbari.org:8082/secure/IssueNavigator.jspa?view=rss&&pid=10000&status=1&sorter/field=issuekey&sorter/order=DESC&sorter/field=priority&sorter/order=DESC&reset=true&decorator=none&os_username=everyone&os_password=guest|columns=type;priority;key;summary;assignee;status} h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). # Load historical OASIS data into SSDS (data and metadata). *This is important but too big for this, separate project* ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). *IMPORTANT EVEN THOUGH DESCOPED* ]]> 4456484 4489252 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms *(8 days - 8 KG)* ## (.5 day) Try to change OASIS to make mooring turns less painful (documentation basically) ## (5 days) Build non-JMS mechanism for users to send data/metadata to SSDS. ## (1 day) Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## (1 day) Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Improve Testing *(5 days - 3 KG, 2MM)* ## (.5 day) Verify (unit tests) that the RecordDescription level parse regular expression works ## (.5 day) Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## (.5 day) Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## (.5 day) Write valid unit test for Object and XMLBuilders ## (.5 day) Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## (.5 day) Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. ## (2 days) Improve Wrapper tests # Documentation *(6 days - 3 KG, 3 MM)* ## (.5 day) Put UML diagram of data model on developer section of web app. ## (.5 day) Finish documenting data packet structure on web pages. ## (5 days) Document Explorer, Admin app and HOOVES h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). # Load historical OASIS data into SSDS (data and metadata). *This is important but too big for this, separate project* ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). *IMPORTANT EVEN THOUGH DESCOPED* ]]> 4456485 10388417 10355689 13172807 Administrative Tools->Internet Information Services, right-click on Default Web Site and select Properties. ## Then on the Home Directory tab, check the box next to Directory browsing. # I created the directory C:\Inetpub\wwwroot\ssdsdata where I will have SSDS store everything. With the data there, it will then be HTTP accessible. # I then browsed to http://jboss.org and downloaded JBoss 4.2.2GA and unzipped it to the C:\bin\jboss-4.2.2GA directory ## I started up JBoss from the command line and noticed that port 8080 had been taken by the SQL services. I shutdown all SQL services, started JBoss so it could claim those ports, then restarted the SQL components. ]]> 13140041 10388404 10355676 4489266 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms *(8 days - 8 KG)* ## (.5 day) Try to change OASIS to make mooring turns less painful (documentation basically) ## (5 days) Build non-JMS mechanism for users to send data/metadata to SSDS. ## (1 day) Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## (1 day) Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Improve Testing *(5 days - 3 KG, 2MM)* ## (.5 day) Verify (unit tests) that the RecordDescription level parse regular expression works ## (.5 day) Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## (.5 day) Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## (.5 day) Write valid unit test for Object and XMLBuilders ## (.5 day) Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## (.5 day) Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. ## (2 days) Improve Wrapper tests # Documentation *(6 days - 3 KG, 3 MM)* ## (.5 day) Put UML diagram of data model on developer section of web app. ## (.5 day) Finish documenting data packet structure on web pages. ## (5 days) Document Explorer, Admin app and HOOVES h3. Tasks that were descoped from 900823 SSDS Hardening Proposal # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). # Load historical OASIS data into SSDS (data and metadata). *This is important but too big for this, separate project* ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). *IMPORTANT EVEN THOUGH DESCOPED* ]]> 4456500 4489263 15 {noformat} to {noformat} 1 {noformat} Which should effectively make the RuminateMDB a singleton which should alleviate our deadlock issues that we were having (at least at Ruminate step). {note}]]> 4456497 13172809 Administrative Tools->Internet Information Services, right-click on Default Web Site and select Properties. ## Then on the Home Directory tab, check the box next to Directory browsing. # I created the directory C:\Inetpub\wwwroot\ssdsdata where I will have SSDS store everything. With the data there, it will then be HTTP accessible. # I then browsed to http://jboss.org and downloaded JBoss 4.2.2GA and unzipped it to the C:\bin\jboss-4.2.2GA directory ## I started up JBoss from the command line and noticed that port 8080 had been taken by the SQL services. I shutdown all SQL services, started JBoss so it could claim those ports, then restarted the SQL components. # I then installed SmartSVN so that I could get the code from the Google SVN server.]]> 13140043 10388406 10355678 4489261 4456495 10388408 10355680 10388410 10355682 4489260 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms *(8 days - 8 KG)* ## (.5 day) Try to change OASIS to make mooring turns less painful (documentation basically) ## (5 days) Build non-JMS mechanism for users to send data/metadata to SSDS. ## (1 day) Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## (1 day) Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Improve Testing *(5 days - 3 KG, 2MM)* ## (.5 day) Verify (unit tests) that the RecordDescription level parse regular expression works ## (.5 day) Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## (.5 day) Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## (.5 day) Write valid unit test for Object and XMLBuilders ## (.5 day) Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## (.5 day) Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. ## (2 days) Improve Wrapper tests # Documentation *(6 days - 3 KG, 3 MM)* ## (.5 day) Put UML diagram of data model on developer section of web app. ## (.5 day) Finish documenting data packet structure on web pages. ## (5 days) Document Explorer, Admin app and HOOVES h3. Descoped # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). # Load historical OASIS data into SSDS (data and metadata). *This is important but too big for this, separate project* ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). *IMPORTANT EVEN THOUGH DESCOPED* ]]> 4456493 4489273 4456509 10388396 10355668 4489272 4456508 10388398 10355670 10388400 10355672 10388402 10355674 10945032 10912277 10945033 10912278 10945036 10912281 10945037 10912282 10945039 10912284 10945040 10912285 10945041 10912286 10945044 10912289 4489239 4456471 10945043 10912288 4489242 4456474 10945046 10912291 4489241 4456473 10945045 10912290 4489236 4456468 10945048 10912293 4489235 4456467 10945050 10912295 4489237 4456469 4489247 4456480 4489250 4456483 4489249 4456482 4489244 4456477 4489245 4456478 10945095 10912348 10945094 10912347 10945103 0, the service will consider any time between // samples that is less than the specified gap plus this margin, // it will assume that it is not a gap condition. This is to // prevent false positives when the sample timestamps aren't exactly // on the interval. String gapSpec, // There are two ways to specify a gap: // "serviceCalculated" or "userSpecified" Long numberOfRecords, // If the call specifies SERVICE_CALCULATED and this // is greater than 0, the service will use 'numberOfRecords' // most recent records of the specified RecordType // in calculating the average time between samples Date intervalCalcStartWindow, // This is the date that starts the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this is ignored. Date intervalCalcEndWindow, // This is the date that ends the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this will be used as // the endtime and then the service will use the numberOfRecords // before this time as the data to calculate the average // time interval. Long gapInMillis // If the gapSpec is USER_SPECIFIED, then the service will use // this number of milliseconds as the gap for identifying gaps. ) {code} With a return that has the format of: Properties Objects with properties: ||Property Name||Value|| |lastPacketDateTime|This is the date and time of the last packet received| |totalNumberOfRecords|This is the total number of records for the parameters specified| |numberOfFuturePackets|This is the number of packet that appear in the future. This should be zero and if they are not, there could be bad data| |averagSampleIntervalInMillis|This is the number of milliseconds that the service used to find data gaps| |marginInMillis|This is the number of milliseconds as a margin that the service used to find data gaps| |numRecordsSearchedForGaps|This is the number of records that were searched through while trying to find gaps using the gap criteria| |dataGap1Start|This is the date and time of start of the first possible gap in the data| |dataGap1End|This is the date and time of end of the first possible gap in the data| |.|.| |.|.| |.|.| |dataGapNStart|This is the date and time of start of the Nth possible gap in the data| |dataGapNEnd|This is the date and time of end of the Nth possible gap in the data| h5. Java EJB client Under construction h5. REST client If you want to use the REST-style interface the HTTP call would looks something like: {code} http://new-ssds.mbari.org:8080/servlet/DataAccessServlet?objectToInvokeOn=SQLDataStreamRawDataAccess&method=getDataStreamProperties &p1Type=Long&p1Value=1300 &p2Type=Long&p2Value=1 &p3Type=Boolean&p3Value=true &p4Type=Date&p4Value=2008-12-00T00:00:00Z &p5Type=Date&p5Value=2008-01-00T00:00:00Z &p6Type=String&p6Value=timeGap &p7Type=Long&p7Value=5000 &p8Type=String&p8Value=userSpecified &p9Type=Long&p9Value=10 &p10Type=Date&p10Value= &p11Type=Date&p11Value= &p12Type=Long&p12Value=10000 {code} And the return might look something like {code} java.util.Properties{} {code} h5. Web Service Client Under construction {panel} {panel:title=getDataStreamProperties} h5. Background The desire it to have a service that can characterize a DataStream from an instrument. This would allow for easier monitoring of instruments on the network. ||Parameter||Description||Options||Default Value||Required|| |Device ID|The SSDS ID of the device that the user wants information about|Any SSDS ID|N/A|Y| |Number Of Samples To Average|The number of samples back that the service is to use to calculate the average sampling interval|Any Number|10|N| |Check for gaps|Is a flag that tells the service to try to find data gaps based on some average sample interval|True/False|False|N| |Gap Interval|Is the number of milliseconds to use as the sampling interval to search for gaps. If this is specified it will override the average gap calculated by the service|Any number of milliseconds|N/A|N| h5. The Interface The things that the user would want to know are: ||Return||Parameters|| |Date and time of last packet received|* Device ID * RecordType to search for (nothing/default means most recent packet)| |Total Number Of Records|* Device ID * RecordType to search for (nothing/default specified means all packets)| |Data Gaps|* Device ID * RecordType * Time window over data to search for gaps * Gap criteria ** Type of gap *** Time only *** Sequence number only *** Time and sequence number ** Ways to specify gap *** Margin on gap in milliseconds (anything longer than gap + margin will be considered a possible gap) *** Let service calculate gap constraints (calculate average time between sample) **** Number of points to use (points back from most recent packet) **** Or time window to use (start to end time) *** Specify gap constraints **** Gap in milliseconds| So the API interface looks like: {code:title=getDataStreamProperties} // Type of criteria to use for finding gaps public final static String TIME_ONLY_GAP = "timeGap"; public final static String SEQ_ONLY_GAP = "seqGap"; public final static String TIME_SEQ_GAP = "timeSeqGap"; // The method of specifying the gap public final static String SERVICE_CALCULATED = "serviceCalculated"; public final static String USER_SPECIFIED = "userSpecified"; getDataStreamProperties( Long deviceID, // The ID of the Device to get the properties for Long recordType, // The RecordType that will be singled out (devices // can send out more than one RecordType) // 0 = Metadata Packets // 1+ = Device specific record types Boolean checkForGaps, // A Boolean that indicates if the caller wants to // have the service check for data gaps (true means // the service will check for gaps and false/null // means it will not Date startGapCheckWindow, // The start date of the window over which to search for // gaps. Date endGapCheckWindow, // The end date of the window over which to search for gaps. String typeOfGap, // One of three types: "timeGap", "seqGap", "timeSeqGap" Long marginMillis, // This is the number of milliseconds that are used as 'slop' // around the specification for a gap. In other words, if // this is > 0, the service will consider any time between // samples that is less than the specified gap plus this margin, // it will assume that it is not a gap condition. This is to // prevent false positives when the sample timestamps aren't exactly // on the interval. String gapSpec, // There are two ways to specify a gap: // "serviceCalculated" or "userSpecified" Long numberOfRecords, // If the call specifies SERVICE_CALCULATED and this // is greater than 0, the service will use 'numberOfRecords' // most recent records of the specified RecordType // in calculating the average time between samples Date intervalCalcStartWindow, // This is the date that starts the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this is ignored. Date intervalCalcEndWindow, // This is the date that ends the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this will be used as // the endtime and then the service will use the numberOfRecords // before this time as the data to calculate the average // time interval. Long gapInMillis // If the gapSpec is USER_SPECIFIED, then the service will use // this number of milliseconds as the gap for identifying gaps. ) {code} With a return that has the format of: Properties Objects with properties: ||Property Name||Value|| |lastPacketDateTime|This is the date and time of the last packet received| |totalNumberOfRecords|This is the total number of records for the parameters specified| |numberOfFuturePackets|This is the number of packet that appear in the future. This should be zero and if they are not, there could be bad data| |averagSampleIntervalInMillis|This is the number of milliseconds that the service used to find data gaps| |marginInMillis|This is the number of milliseconds as a margin that the service used to find data gaps| |numRecordsSearchedForGaps|This is the number of records that were searched through while trying to find gaps using the gap criteria| |dataGap1Start|This is the date and time of start of the first possible gap in the data| |dataGap1End|This is the date and time of end of the first possible gap in the data| |.|.| |.|.| |.|.| |dataGapNStart|This is the date and time of start of the Nth possible gap in the data| |dataGapNEnd|This is the date and time of end of the Nth possible gap in the data| h5. Java EJB client Under construction h5. REST client If you want to use the REST-style interface the HTTP call would looks something like: {code} http://new-ssds.mbari.org:8080/servlet/DataAccessServlet?objectToInvokeOn=SQLDataStreamRawDataAccess&method=getDataStreamProperties &p1Type=Long&p1Value=1300 &p2Type=Long&p2Value=1 &p3Type=Boolean&p3Value=true &p4Type=Date&p4Value=2008-12-00T00:00:00Z &p5Type=Date&p5Value=2008-01-00T00:00:00Z &p6Type=String&p6Value=timeGap &p7Type=Long&p7Value=5000 &p8Type=String&p8Value=userSpecified &p9Type=Long&p9Value=10 &p10Type=Date&p10Value= &p11Type=Date&p11Value= &p12Type=Long&p12Value=10000 {code} And the return might look something like {code} java.util.Properties{} {code} h5. Web Service Client Under construction {panel}]]> 10912357 9831013 9798260 9831014 9798261 10945101 0, the service will consider any time between // samples that is less than the specified gap plus this margin, // it will assume that it is not a gap condition. This is to // prevent false positives when the sample timestamps aren't exactly // on the interval. String gapSpec, // There are two ways to specify a gap: // "serviceCalculated" or "userSpecified" Long numberOfRecords, // If the call specifies SERVICE_CALCULATED and this // is greater than 0, the service will use 'numberOfRecords' // most recent records of the specified RecordType // in calculating the average time between samples Date intervalCalcStartWindow, // This is the date that starts the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this is ignored. Date intervalCalcEndWindow, // This is the date that ends the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this will be used as // the endtime and then the service will use the numberOfRecords // before this time as the data to calculate the average // time interval. Long gapInMillis // If the gapSpec is USER_SPECIFIED, then the service will use // this number of milliseconds as the gap for identifying gaps. ) {code} With a return that has the format of: Properties Objects with properties: ||Property Name||Value|| |lastPacketDateTime|This is the date and time of the last packet received| |totalNumberOfRecords|This is the total number of records for the parameters specified| |numberOfFuturePackets|This is the number of packet that appear in the future. This should be zero and if they are not, there could be bad data| |averagSampleIntervalInMillis|This is the number of milliseconds that the service used to find data gaps| |marginInMillis|This is the number of milliseconds as a margin that the service used to find data gaps| |numRecordsSearchedForGaps|This is the number of records that were searched through while trying to find gaps using the gap criteria| |dataGap1Start|This is the date and time of start of the first possible gap in the data| |dataGap1End|This is the date and time of end of the first possible gap in the data| |.|.| |.|.| |.|.| |dataGapNStart|This is the date and time of start of the Nth possible gap in the data| |dataGapNEnd|This is the date and time of end of the Nth possible gap in the data| h5. Java EJB client Under construction h5. REST client If you want to use the REST-style interface the HTTP call would looks something like: {code} http://new-ssds.mbari.org:8080/servlet/DataAccessServlet?objectToInvokeOn=SQLDataStreamRawDataAccess&method=getDataStreamProperties &p1Type=Long&p1Value=1300 &p2Type=Long&p2Value=1 &p3Type=Boolean&p3Value=true &p4Type=Date&p4Value=2008-12-00T00:00:00Z &p5Type=Date&p5Value=2008-01-00T00:00:00Z &p6Type=String&p6Value=timeGap &p7Type=Long&p7Value=5000 &p8Type=String&p8Value=userSpecified &p9Type=Long&p9Value=10 &p10Type=Date&p10Value= &p11Type=Date&p11Value= &p12Type=Long&p12Value=10000 {code} And the return might look something like {code} java.util.Properties{} {code} h5. Web Service Client Under construction {panel}]]> 10912354 10388419 10355691 8388769 8356008 8388768 8356007 15 17 8388767 8356006 14 16 13 15 8388765 8356004 12 14 8388764 8356003 8388762 8356001 8388760 8355999 7 9 8388758 8355997 8388756 8355995 33 35 8388753 8355992 34 36 8388754 8355993 8388751 0, the service will consider any time between // samples that is less than the specified gap plus this margin, // it will assume that it is not a gap condition. This is to // prevent false positives when the sample timestamps aren't exactly // on the interval. String gapSpec, // There are two ways to specify a gap: // SERVICE_CALCULATED or USER_SPECIFIED Long numberOfRecords, // If the call specifies SERVICE_CALCULATED and this // is greater than 0, the service will use 'numberOfRecords' // most recent records of the specified RecordType // in calculating the average time between samples Date intervalCalcStartWindow, // This is the date that starts the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfPoints is specified, this is ignored. Date intervalCalcEndWindow, // This is the date that ends the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this will be used as // the endtime and then the service will use the numberOfRecords // before this time as the data to calculate the average // time interval. Long gapInMillis // If the gapSpec is USER_SPECIFIED, then the service will use // this number of milliseconds as the gap for identifying gaps. ) {code} With a return that has the format of: Properties Objects with properties: ||Property Name||Value|| |lastPacketDateTime|This is the date and time of the last packet received| |totalNumberOfRecords|This is the total number of records for the parameters specified| |numberOfFuturePackets|This is the number of packet that appear in the future. This should be zero and if they are not, there could be bad data| |averagSampleIntervalInMillis|This is the number of milliseconds that the service used to find data gaps| |marginInMillis|This is the number of milliseconds as a margin that the service used to find data gaps| |dataGap1Start|This is the date and time of start of the first possible gap in the data| |dataGap1End|This is the date and time of end of the first possible gap in the data| |.|.| |.|.| |.|.| |dataGapNStart|This is the date and time of start of the Nth possible gap in the data| |dataGapNEnd|This is the date and time of end of the Nth possible gap in the data| {panel}]]> 8355990 32 34 29 31 8388749 0, the service will consider any time between // samples that is less than the specified gap plus this margin, // it will assume that it is not a gap condition. This is to // prevent false positives when the sample timestamps aren't exactly // on the interval. String gapSpec, // There are two ways to specify a gap: // SERVICE_CALCULATED or USER_SPECIFIED Long numberOfRecords, // If the call specifies SERVICE_CALCULATED and this // is greater than 0, the service will use 'numberOfRecords' // most recent records of the specified RecordType // in calculating the average time between samples Date intervalCalcStartWindow, // This is the date that starts the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfPoints is specified, this is ignored. Date intervalCalcEndWindow, // This is the date that ends the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this will be used as // the endtime and then the service will use the numberOfRecords // before this time as the data to calculate the average // time interval. Long gapInMillis // If the gapSpec is USER_SPECIFIED, then the service will use // this number of milliseconds as the gap for identifying gaps. ) {code} With a return that has the format of: Properties Objects with properties: ||Property Name||Value|| |lastPacketDateTime|This is the date and time of the last packet received| |totalNumberOfRecords|This is the total number of records for the parameters specified| |averagSampleIntervalInMillis|This is the number of milliseconds that the service used to find data gaps| |marginInMillis|This is the number of milliseconds as a margin that the service used to find data gaps| |dataGap1Start|This is the date and time of start of the first possible gap in the data| |dataGap1End|This is the date and time of end of the first possible gap in the data| |.|.| |.|.| |.|.| |dataGapNStart|This is the date and time of start of the Nth possible gap in the data| |dataGapNEnd|This is the date and time of end of the Nth possible gap in the data| {panel}]]> 8355988 30 32 8388747 0, the service will consider any time between // samples that is less than the specified gap plus this margin, // it will assume that it is not a gap condition. This is to // prevent false positives when the sample timestamps aren't exactly // on the interval. String gapSpec, // There are two ways to specify a gap: // SERVICE_CALCULATED or USER_SPECIFIED Long numberOfRecords, // If the call specifies SERVICE_CALCULATED and this // is greater than 0, the service will use 'numberOfRecords' // most recent records of the specified RecordType // in calculating the average time between samples Date intervalCalcStartWindow, // This is the date that starts the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfPoints is specified, this is ignored. Date intervalCalcEndWindow, // This is the date that ends the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this will be used as // the endtime and then the service will use the numberOfRecords // before this time as the data to calculate the average // time interval. Long gapInMillis // If the gapSpec is USER_SPECIFIED, then the service will use // this number of milliseconds as the gap for identifying gaps. ) {code} With a return that has the format of: Properties Objects with properties: ||Property Name||Value|| |lastPacketDateTime|This is the date and time of the last packet received| |totalNumberOfRecords|This is the total number of records for the parameters specified| |averagSampleIntervalInMillis|This is the number of milliseconds that the service used to find data gaps| |marginInMillis|This is the number of milliseconds as a margin that the service used to find data gaps| |dataGap1Start|This is the date and time of start of the first possible gap in the data| |dataGap1End|This is the date and time of end of the first possible gap in the data| |.|.| |.|.| |.|.| |dataGapNStart|This is the date and time of start of the Nth possible gap in the data| |dataGapNEnd|This is the date and time of end of the Nth possible gap in the data| {panel}]]> 8355986 25 27 3670788 3638041 26 28 8388746 0, the service will consider any time between // samples that is less than the specified gap plus this margin, // it will assume that it is not a gap condition. This is to // prevent false positives when the sample timestamps aren't exactly // on the interval. String gapSpec, // There are two ways to specify a gap: // SERVICE_CALCULATED or USER_SPECIFIED Long numberOfRecords, // If the call specifies SERVICE_CALCULATED and this // is greater than 0, the service will use 'numberOfRecords' // most recent records of the specified RecordType // in calculating the average time between samples Date intervalCalcStartWindow, // This is the date that starts the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfPoints is specified, this is ignored. Date intervalCalcEndWindow, // This is the date that ends the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this will be used as // the endtime and then the service will use the numberOfRecords // before this time as the data to calculate the average // time interval. Long gapInMillis // If the gapSpec is USER_SPECIFIED, then the service will use // this number of milliseconds as the gap for identifying gaps. ) {code} With a return that has the format of: Properties Objects with properties: ||Property Name||Value|| |lastPacketDateTime|This is the date and time of the last packet received| |totalNumberOfRecords|This is the total number of records for the parameters specified| |dataGap1Start|This is the date and time of start of the first possible gap in the data| |dataGap1End|This is the date and time of end of the first possible gap in the data| |.|.| |.|.| |.|.| |dataGapNStart|This is the date and time of start of the Nth possible gap in the data| |dataGapNEnd|This is the date and time of end of the Nth possible gap in the data| {panel}]]> 8355985 23 25 3670790 3638043 8388744 8355983 3670792 3638045 3670794 3638047 11273163 Administrative Tools->Internet Information Services, right-click on Default Web Site and select Properties. ## Then on the Home Directory tab, check the box next to Directory browsing. # I created the directory C:\Inetpub\wwwroot\ssdsdata where I will have SSDS store everything. With the data there, it will then be HTTP accessible. # I then browsed to http://jboss.org and downloaded JBoss 4.2.2GA and unzipped it to the C:\bin\jboss-4.2.2GA directory ]]> 11240426 11273158 11240421 39 41 42 44 11273156 11240419 41 43 11273161 Administrative Tools->Internet Information Services, right-click on Default Web Site and select Properties. ## Then on the Home Directory tab, check the box next to Directory browsing. # I created the directory C:\Inetpub\wwwroot\ssdsdata where I will have SSDS store everything. With the data there, it will then be HTTP accessible. # I then browsed to http://jboss.org and downloaded JBoss 4.2.2GA. ]]> 11240424 11273159 Administrative Tools->Internet Information Services, right-click on Default Web Site and select Properties. ## Then on the Home Directory tab, check the box next to Directory browsing. # I created the directory C:\Inetpub\wwwroot\ssdsdata where I will have SSDS store everything. With the data there, it will then be HTTP accessible.]]> 11240422 38 40 37 39 17268874 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the change log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is [http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log] # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: [http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED] To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:`PSAL,return=lend`/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC !-> LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:5182/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the first 3 temperature values from the 1m and 10m input CTD data and the first 3 times of the output axis: {noformat} yes? list/l=1:3/d=1 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1338 at original sampling intervals FILENAME : m1_ctd0001_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 1 122W 1 27-OCT-2010 21:08:43 / 1: 14.01 27-OCT-2010 21:22:27 / 2: 14.10 27-OCT-2010 21:28:23 / 3: 14.10 yes? list/l=1:3/d=2 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 27-OCT-2010 20:11:17 / 1: 13.38 27-OCT-2010 20:21:16 / 2: 13.59 27-OCT-2010 20:31:16 / 3: 13.45 yes? show/l=1:3 axis time name axis # pts start end TIME TIME 5181 r 27-OCT-2010 20:30 31-MAY-2011 16:30 T0 = 01-JAN-1950 00:00:00 Axis span (to cell edges) = 215.875 L T TBOX TBOXLO TSTEP (DAYS) 1> 27-OCT-2010 20:30:00 0.0416667 27-OCT-2010 20:00:00 22214.85 2> 27-OCT-2010 21:30:00 0.0416667 27-OCT-2010 21:00:00 22214.9 3> 27-OCT-2010 22:30:00 0.0416667 27-OCT-2010 22:00:00 22214.94 {noformat} The problem seems to be that the 10m data begin at 20:11:17 before the 1m data at 21:08:43. The first write of the 1m data to the output netCDF file gives the file its shape with the KLIMITS= and LLIMITS= options. Writing the 10m data, which begins in before the bounds that were defined violates Ferret's axis ordering logic. To test this hypothesis edit the SAVE statement in the temporary .jnl file to make the start time for the 10m data 21:00, execute it and then examine the output file: {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 21:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc (Another ferret session) > ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 31-May-11 12:19 yes? use "/tmp/OS_MBARI-M1_20101027_R_TS.nc" yes? list/l=1:10 temp VARIABLE : Hourly sea_water_temperature (celsius) FILENAME : OS_MBARI-M1_20101027_R_TS.nc FILEPATH : /tmp/ SUBSET : 11 by 10 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 27-OCT-2010 21:30 / 1: 14.09 13.60 .... .... .... .... .... .... .... .... .... 27-OCT-2010 22:30 / 2: 14.17 13.62 .... .... .... .... .... .... .... .... .... 27-OCT-2010 23:30 / 3: 14.34 13.63 .... .... .... .... .... .... .... .... .... 28-OCT-2010 00:30 / 4: 14.28 13.69 .... .... .... .... .... .... .... .... .... 28-OCT-2010 01:30 / 5: 14.22 13.72 .... .... .... .... .... .... .... .... .... 28-OCT-2010 02:30 / 6: 14.21 13.78 .... .... .... .... .... .... .... .... .... 28-OCT-2010 03:30 / 7: 14.21 13.76 .... .... .... .... .... .... .... .... .... 28-OCT-2010 04:30 / 8: 14.09 13.79 .... .... .... .... .... .... .... .... .... 28-OCT-2010 05:30 / 9: 14.01 13.77 .... .... .... .... .... .... .... .... .... 28-OCT-2010 06:30 / 10: 13.89 13.76 .... .... .... .... .... .... .... .... .... yes? {noformat} O.K.\! That looks good. The 10m data got written. Now to fix it for production. This deployment was a special case where the 1m data start after the 10m data. Typically, it's most convenient to have all of the microcats have the same deployment start time. This makes the follow on data processing much simpler. h4. B. Fixing the start times for all the child instrument deployments All of the deployment start times in SSDS_METADTA for the M1 201010 deployment were adjusted to be the same as the start time for the Mooring deployment. Performing this step is now part of the [standard operating procedure for perming mooring turns|SSDS:OASIS Mooring turn]. Once SSDS_METADATA has been updated the individual instrument netCDF files must be created by DStoNetCDF.pl before combineTS.pl in run. h4. C. Examining the gridding method There are still some problems with gappy data at 40m and below in the \_TS file. To debug these we'll make another copy of the OS_MBARI-M1_20101027_R_TS.jnl to /tmp and edit it to go through the processing of the 40m data saving the data to a netCDF file in /tmp so as not to disturb the production processing. We also comment out all of the "CANCEL DATA 1" statements so that we can examine data from the files. This file is executed and we see no errors. We need to examine the input data and the Ferret variables that are constructed for the gridding. Here are the commands from the .jnl that perform the the gridding for the 40m data: {noformat} ! ! Remove temperature outliers before gridding ! LET Temperature_QFLAG = IF Temperature GT 2 AND Temperature LT 20 THEN 1 ELSE (-99999) SET VAR/BAD=-99999 Temperature_QFLAG LET Temperature_QC = Temperature_QFLAG * Temperature ! ! Compute my own mean of the data, making sure to assign missing values in the gaps in the Gap FLAG ! Allow at least 1 data point in each destination cell - there are usually 6 for 10 minute data ! LET Temperature_GOOD = Temperature_QC[gt=TIME@SUM] / Temperature_QC[gt=TIME@NGD] LET Temperature_GFLAG = IF Temperature[gt=TIME@NGD,gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] LT 1 THEN (-99999) ELSE 1 SET VAR/BAD=-99999 Temperature_GFLAG LET TEMP = Temperature_GFLAG * Temperature_GOOD[gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] {noformat} Here are some sample Ferret commands for plotting these data from the session that just executed the OS_MBARI-M1_20101027_R_TS.jnl script: {noformat} yes? go "OS_MBARI-M1_20101027_R_TS.jnl" yes? set region/t=1-may-2011:1-jun-2011 yes? plot Temperature_QC yes? plot/ov/symbol=1 temperature yes? set win 2 yes? plot Temperature_QC[gt=TIME@NGD] yes? plot TEMPERATURE_QC[GT=TIME@SUM] {noformat} These last two plot commands indicate the source of the problem of the gappy data from the inductive modem microcats. Another big clue is the comment in the .jnl file referring to the 10-minute input data and having at least 6 good values within each gridding cell. As can be seen in the plot we often have less than 1 for the @NGD value for the 40m data. These data are hourly and we should be using a different gridding algorithm for them. !m1_ngd_sum.gif|thumbnail! The October 2010 M1 deployment was the first one where all the inductive modem connected CTDs were logged as separate instruments, and the first deployment where the serial microcat processing software was used to process the inductive modem microcat data. The fix for this problem appears to be to do a different gridding for the hourly data from 40m and below. This change was implemented in the code on 2 June 2011 using the Ferret same gridding transform (@MAX) that was previously used for the inductive modem data processing.  Here is the after figure: !wind_SEA_WATER_TEMPERATURE_HR_last30_after.gif|thumbnail! The subsurface data looks a lot better, but there are still some gaps.  If the gaps extend across the whole water column encompassing both the inductive modem and serial microcats then perhaps the gap is due to an interruption in data processing through oasisToSSDS.  This can be remedied by reprocessing the telemetered data by following the README instructions in /u/ssdsadmin/dev/DPforSSDS/oasis/scripts on elvis. h4. D. Testing the results of the gridding It is worthwhile to do another check on the quality of the gridding that is now being performed.  The following Ferret exercice will show the input data compared to the gridded data. {noformat} USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" SET REGION/T="29-May-2011 18:13":"02-Jun-2011 18:13" shade SEA_WATER_TEMPERATURE_HR set win 2 plot/k=1/vlimits=7:13 SEA_WATER_TEMPERATURE_HR repeat/k=2:11 plot/ov SEA_WATER_TEMPERATURE_HR {noformat} Produces these images showing missing data, especially from the 20m and 40m microcats: !m1_shade.gif|thumbnail! !m1_lines.gif|thumbnail! Let's compare the original instruments date from 20m and 40m to the gridded product, using the same Ferret session from above:" {noformat} use "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0020_20101027_original.nc" plot/symbol=1 temperature[d=2] plot/ov SEA_WATER_TEMPERATURE_HR[k=3,d=1] use "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0040_20101027_original.nc" plot/symbol=1 temperature[d=3] plot/ov SEA_WATER_TEMPERATURE_HR[k=4,d=1] {noformat} Gives: !m1_compare_20.gif|thumbnail! !m1_compare_40.gif|thumbnail! We obviously still have some gridding issues... Now is the time to iterate using our method of starting with the .jnl file that aggregates and grids the data to the OceanSITES \_TS.nc file. \\]]> 17236110 17268872 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the change log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is [http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log] # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: [http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED] To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:`PSAL,return=lend`/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC !-> LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:5182/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the first 3 temperature values from the 1m and 10m input CTD data and the first 3 times of the output axis: {noformat} yes? list/l=1:3/d=1 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1338 at original sampling intervals FILENAME : m1_ctd0001_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 1 122W 1 27-OCT-2010 21:08:43 / 1: 14.01 27-OCT-2010 21:22:27 / 2: 14.10 27-OCT-2010 21:28:23 / 3: 14.10 yes? list/l=1:3/d=2 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 27-OCT-2010 20:11:17 / 1: 13.38 27-OCT-2010 20:21:16 / 2: 13.59 27-OCT-2010 20:31:16 / 3: 13.45 yes? show/l=1:3 axis time name axis # pts start end TIME TIME 5181 r 27-OCT-2010 20:30 31-MAY-2011 16:30 T0 = 01-JAN-1950 00:00:00 Axis span (to cell edges) = 215.875 L T TBOX TBOXLO TSTEP (DAYS) 1> 27-OCT-2010 20:30:00 0.0416667 27-OCT-2010 20:00:00 22214.85 2> 27-OCT-2010 21:30:00 0.0416667 27-OCT-2010 21:00:00 22214.9 3> 27-OCT-2010 22:30:00 0.0416667 27-OCT-2010 22:00:00 22214.94 {noformat} The problem seems to be that the 10m data begin at 20:11:17 before the 1m data at 21:08:43. The first write of the 1m data to the output netCDF file gives the file its shape with the KLIMITS= and LLIMITS= options. Writing the 10m data, which begins in before the bounds that were defined violates Ferret's axis ordering logic. To test this hypothesis edit the SAVE statement in the temporary .jnl file to make the start time for the 10m data 21:00, execute it and then examine the output file: {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 21:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc (Another ferret session) > ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 31-May-11 12:19 yes? use "/tmp/OS_MBARI-M1_20101027_R_TS.nc" yes? list/l=1:10 temp VARIABLE : Hourly sea_water_temperature (celsius) FILENAME : OS_MBARI-M1_20101027_R_TS.nc FILEPATH : /tmp/ SUBSET : 11 by 10 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 27-OCT-2010 21:30 / 1: 14.09 13.60 .... .... .... .... .... .... .... .... .... 27-OCT-2010 22:30 / 2: 14.17 13.62 .... .... .... .... .... .... .... .... .... 27-OCT-2010 23:30 / 3: 14.34 13.63 .... .... .... .... .... .... .... .... .... 28-OCT-2010 00:30 / 4: 14.28 13.69 .... .... .... .... .... .... .... .... .... 28-OCT-2010 01:30 / 5: 14.22 13.72 .... .... .... .... .... .... .... .... .... 28-OCT-2010 02:30 / 6: 14.21 13.78 .... .... .... .... .... .... .... .... .... 28-OCT-2010 03:30 / 7: 14.21 13.76 .... .... .... .... .... .... .... .... .... 28-OCT-2010 04:30 / 8: 14.09 13.79 .... .... .... .... .... .... .... .... .... 28-OCT-2010 05:30 / 9: 14.01 13.77 .... .... .... .... .... .... .... .... .... 28-OCT-2010 06:30 / 10: 13.89 13.76 .... .... .... .... .... .... .... .... .... yes? {noformat} O.K.\! That looks good. The 10m data got written. Now to fix it for production. This deployment was a special case where the 1m data start after the 10m data. Typically, it's most convenient to have all of the microcats have the same deployment start time. This makes the follow on data processing much simpler. All of the deployment start times in SSDS_METADTA for the M1 201010 deployment were adjusted to be the same as the start time for the Mooring deployment. Performing this step is now part of the [standard operating procedure for perming mooring turns|SSDS:OASIS Mooring turn]. Once SSDS_METADATA has been updated the individual instrument netCDF files must be created by DStoNetCDF.pl before combineTS.pl in run. There are still some problems with gappy data at 40m and below in the \_TS file. To debug these we'll make another copy of the OS_MBARI-M1_20101027_R_TS.jnl to /tmp and edit it to go through the processing of the 40m data saving the data to a netCDF file in /tmp so as not to disturb the production processing. We also comment out all of the "CANCEL DATA 1" statements so that we can examine data from the files. This file is executed and we see no errors. We need to examine the input data and the Ferret variables that are constructed for the gridding. Here are the commands from the .jnl that perform the the gridding for the 40m data: {noformat} ! ! Remove temperature outliers before gridding ! LET Temperature_QFLAG = IF Temperature GT 2 AND Temperature LT 20 THEN 1 ELSE (-99999) SET VAR/BAD=-99999 Temperature_QFLAG LET Temperature_QC = Temperature_QFLAG * Temperature ! ! Compute my own mean of the data, making sure to assign missing values in the gaps in the Gap FLAG ! Allow at least 1 data point in each destination cell - there are usually 6 for 10 minute data ! LET Temperature_GOOD = Temperature_QC[gt=TIME@SUM] / Temperature_QC[gt=TIME@NGD] LET Temperature_GFLAG = IF Temperature[gt=TIME@NGD,gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] LT 1 THEN (-99999) ELSE 1 SET VAR/BAD=-99999 Temperature_GFLAG LET TEMP = Temperature_GFLAG * Temperature_GOOD[gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] {noformat} Here are some sample Ferret commands for plotting these data from the session that just executed the OS_MBARI-M1_20101027_R_TS.jnl script: {noformat} yes? go "OS_MBARI-M1_20101027_R_TS.jnl" yes? set region/t=1-may-2011:1-jun-2011 yes? plot Temperature_QC yes? plot/ov/symbol=1 temperature yes? set win 2 yes? plot Temperature_QC[gt=TIME@NGD] yes? plot TEMPERATURE_QC[GT=TIME@SUM] {noformat} These last two plot commands indicate the source of the problem of the gappy data from the inductive modem microcats. Another big clue is the comment in the .jnl file referring to the 10-minute input data and having at least 6 good values within each gridding cell. As can be seen in the plot we often have less than 1 for the @NGD value for the 40m data. These data are hourly and we should be using a different gridding algorithm for them. !m1_ngd_sum.gif|thumbnail! The October 2010 M1 deployment was the first one where all the inductive modem connected CTDs were logged as separate instruments, and the first deployment where the serial microcat processing software was used to process the inductive modem microcat data. The fix for this problem appears to be to do a different gridding for the hourly data from 40m and below. This change was implemented in the code on 2 June 2011 using the Ferret same gridding transform (@MAX) that was previously used for the inductive modem data processing.  Here is the after figure: !wind_SEA_WATER_TEMPERATURE_HR_last30_after.gif|thumbnail! The subsurface data looks a lot better, but there are still some gaps.  If the gaps extend across the whole water column encompassing both the inductive modem and serial microcats then perhaps the gap is due to an interruption in data processing through oasisToSSDS.  This can be remedied by reprocessing the telemetered data by following the README instructions in /u/ssdsadmin/dev/DPforSSDS/oasis/scripts on elvis.   It is worthwhile to do another check on the quality of the gridding that is now being performed.  The following Ferret exercice will show the input data compared to the gridded data. {noformat} USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" SET REGION/T="29-May-2011 18:13":"02-Jun-2011 18:13" shade SEA_WATER_TEMPERATURE_HR set win 2 plot/k=1/vlimits=7:13 SEA_WATER_TEMPERATURE_HR repeat/k=2:11 plot/ov SEA_WATER_TEMPERATURE_HR {noformat} Produces these images showing missing data, especially from the 20m and 40m microcats: !m1_shade.gif|thumbnail! !m1_lines.gif|thumbnail! Let's compare the original instruments date from 20m and 40m to the gridded product, using the same Ferret session from above:" {noformat} use "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0020_20101027_original.nc" plot/symbol=1 temperature[d=2] plot/ov SEA_WATER_TEMPERATURE_HR[k=3,d=1] use "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0040_20101027_original.nc" plot/symbol=1 temperature[d=3] plot/ov SEA_WATER_TEMPERATURE_HR[k=4,d=1] {noformat} Gives: !m1_compare_20.gif|thumbnail! !m1_compare_40.gif|thumbnail! We obviously still have some gridding issues... Now is the time to iterate using our method of starting with the .jnl file that aggregates and grids the data to the OceanSITES _TS.nc file. \\]]> 17236108 8388779 8356018 59 61 17268870 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the change log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is [http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log] # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: [http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED] To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:`PSAL,return=lend`/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC !-> LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:5182/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the first 3 temperature values from the 1m and 10m input CTD data and the first 3 times of the output axis: {noformat} yes? list/l=1:3/d=1 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1338 at original sampling intervals FILENAME : m1_ctd0001_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 1 122W 1 27-OCT-2010 21:08:43 / 1: 14.01 27-OCT-2010 21:22:27 / 2: 14.10 27-OCT-2010 21:28:23 / 3: 14.10 yes? list/l=1:3/d=2 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 27-OCT-2010 20:11:17 / 1: 13.38 27-OCT-2010 20:21:16 / 2: 13.59 27-OCT-2010 20:31:16 / 3: 13.45 yes? show/l=1:3 axis time name axis # pts start end TIME TIME 5181 r 27-OCT-2010 20:30 31-MAY-2011 16:30 T0 = 01-JAN-1950 00:00:00 Axis span (to cell edges) = 215.875 L T TBOX TBOXLO TSTEP (DAYS) 1> 27-OCT-2010 20:30:00 0.0416667 27-OCT-2010 20:00:00 22214.85 2> 27-OCT-2010 21:30:00 0.0416667 27-OCT-2010 21:00:00 22214.9 3> 27-OCT-2010 22:30:00 0.0416667 27-OCT-2010 22:00:00 22214.94 {noformat} The problem seems to be that the 10m data begin at 20:11:17 before the 1m data at 21:08:43. The first write of the 1m data to the output netCDF file gives the file its shape with the KLIMITS= and LLIMITS= options. Writing the 10m data, which begins in before the bounds that were defined violates Ferret's axis ordering logic. To test this hypothesis edit the SAVE statement in the temporary .jnl file to make the start time for the 10m data 21:00, execute it and then examine the output file: {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 21:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc (Another ferret session) > ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 31-May-11 12:19 yes? use "/tmp/OS_MBARI-M1_20101027_R_TS.nc" yes? list/l=1:10 temp VARIABLE : Hourly sea_water_temperature (celsius) FILENAME : OS_MBARI-M1_20101027_R_TS.nc FILEPATH : /tmp/ SUBSET : 11 by 10 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 27-OCT-2010 21:30 / 1: 14.09 13.60 .... .... .... .... .... .... .... .... .... 27-OCT-2010 22:30 / 2: 14.17 13.62 .... .... .... .... .... .... .... .... .... 27-OCT-2010 23:30 / 3: 14.34 13.63 .... .... .... .... .... .... .... .... .... 28-OCT-2010 00:30 / 4: 14.28 13.69 .... .... .... .... .... .... .... .... .... 28-OCT-2010 01:30 / 5: 14.22 13.72 .... .... .... .... .... .... .... .... .... 28-OCT-2010 02:30 / 6: 14.21 13.78 .... .... .... .... .... .... .... .... .... 28-OCT-2010 03:30 / 7: 14.21 13.76 .... .... .... .... .... .... .... .... .... 28-OCT-2010 04:30 / 8: 14.09 13.79 .... .... .... .... .... .... .... .... .... 28-OCT-2010 05:30 / 9: 14.01 13.77 .... .... .... .... .... .... .... .... .... 28-OCT-2010 06:30 / 10: 13.89 13.76 .... .... .... .... .... .... .... .... .... yes? {noformat} O.K.\! That looks good. The 10m data got written. Now to fix it for production. This deployment was a special case where the 1m data start after the 10m data. Typically, it's most convenient to have all of the microcats have the same deployment start time. This makes the follow on data processing much simpler. All of the deployment start times in SSDS_METADTA for the M1 201010 deployment were adjusted to be the same as the start time for the Mooring deployment. Performing this step is now part of the [standard operating procedure for perming mooring turns|SSDS:OASIS Mooring turn]. Once SSDS_METADATA has been updated the individual instrument netCDF files must be created by DStoNetCDF.pl before combineTS.pl in run. There are still some problems with gappy data at 40m and below in the \_TS file. To debug these we'll make another copy of the OS_MBARI-M1_20101027_R_TS.jnl to /tmp and edit it to go through the processing of the 40m data saving the data to a netCDF file in /tmp so as not to disturb the production processing. We also comment out all of the "CANCEL DATA 1" statements so that we can examine data from the files. This file is executed and we see no errors. We need to examine the input data and the Ferret variables that are constructed for the gridding. Here are the commands from the .jnl that perform the the gridding for the 40m data: {noformat} ! ! Remove temperature outliers before gridding ! LET Temperature_QFLAG = IF Temperature GT 2 AND Temperature LT 20 THEN 1 ELSE (-99999) SET VAR/BAD=-99999 Temperature_QFLAG LET Temperature_QC = Temperature_QFLAG * Temperature ! ! Compute my own mean of the data, making sure to assign missing values in the gaps in the Gap FLAG ! Allow at least 1 data point in each destination cell - there are usually 6 for 10 minute data ! LET Temperature_GOOD = Temperature_QC[gt=TIME@SUM] / Temperature_QC[gt=TIME@NGD] LET Temperature_GFLAG = IF Temperature[gt=TIME@NGD,gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] LT 1 THEN (-99999) ELSE 1 SET VAR/BAD=-99999 Temperature_GFLAG LET TEMP = Temperature_GFLAG * Temperature_GOOD[gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] {noformat} Here are some sample Ferret commands for plotting these data from the session that just executed the OS_MBARI-M1_20101027_R_TS.jnl script: {noformat} yes? go "OS_MBARI-M1_20101027_R_TS.jnl" yes? set region/t=1-may-2011:1-jun-2011 yes? plot Temperature_QC yes? plot/ov/symbol=1 temperature yes? set win 2 yes? plot Temperature_QC[gt=TIME@NGD] yes? plot TEMPERATURE_QC[GT=TIME@SUM] {noformat} These last two plot commands indicate the source of the problem of the gappy data from the inductive modem microcats. Another big clue is the comment in the .jnl file referring to the 10-minute input data and having at least 6 good values within each gridding cell. As can be seen in the plot we often have less than 1 for the @NGD value for the 40m data. These data are hourly and we should be using a different gridding algorithm for them. !m1_ngd_sum.gif|thumbnail! The October 2010 M1 deployment was the first one where all the inductive modem connected CTDs were logged as separate instruments, and the first deployment where the serial microcat processing software was used to process the inductive modem microcat data. The fix for this problem appears to be to do a different gridding for the hourly data from 40m and below. This change was implemented in the code on 2 June 2011 using the Ferret same gridding transform (@MAX) that was previously used for the inductive modem data processing.  Here is the after figure: !wind_SEA_WATER_TEMPERATURE_HR_last30_after.gif|thumbnail! The subsurface data looks a lot better, but there are still some gaps.  If the gaps extend across the whole water column encompassing both the inductive modem and serial microcats then perhaps the gap is due to an interruption in data processing through oasisToSSDS.  This can be remedied by reprocessing the telemetered data by following the README instructions in /u/ssdsadmin/dev/DPforSSDS/oasis/scripts on elvis.   It is worthwhile to do another check on the quality of the gridding that is now being performed.  The following Ferret exercice will show the input data compared to the gridded data. {noformat} USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" SET REGION/T="29-May-2011 18:13":"02-Jun-2011 18:13" shade SEA_WATER_TEMPERATURE_HR set win 2 plot/k=1/vlimits=7:13 SEA_WATER_TEMPERATURE_HR repeat/k=2:11 plot/ov SEA_WATER_TEMPERATURE_HR {noformat} Produces these images showing missing data, especially from the 20m and 40m microcats: !m1_shade.gif|thumbnail! !m1_lines.gif|thumbnail! Let's compare the original instruments date from 20m and 40m to the gridded product, using the same Ferret session from above:" {noformat} use "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0020_20101027_original.nc" plot/symbol=1 temperature[d=2] plot/ov SEA_WATER_TEMPERATURE_HR[k=3,d=1] {noformat} Gives:     \\]]> 17236106 60 62 8388781 8356020 61 63 17268868 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the change log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is [http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log] # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: [http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED] To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:`PSAL,return=lend`/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC !-> LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:5182/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the first 3 temperature values from the 1m and 10m input CTD data and the first 3 times of the output axis: {noformat} yes? list/l=1:3/d=1 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1338 at original sampling intervals FILENAME : m1_ctd0001_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 1 122W 1 27-OCT-2010 21:08:43 / 1: 14.01 27-OCT-2010 21:22:27 / 2: 14.10 27-OCT-2010 21:28:23 / 3: 14.10 yes? list/l=1:3/d=2 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 27-OCT-2010 20:11:17 / 1: 13.38 27-OCT-2010 20:21:16 / 2: 13.59 27-OCT-2010 20:31:16 / 3: 13.45 yes? show/l=1:3 axis time name axis # pts start end TIME TIME 5181 r 27-OCT-2010 20:30 31-MAY-2011 16:30 T0 = 01-JAN-1950 00:00:00 Axis span (to cell edges) = 215.875 L T TBOX TBOXLO TSTEP (DAYS) 1> 27-OCT-2010 20:30:00 0.0416667 27-OCT-2010 20:00:00 22214.85 2> 27-OCT-2010 21:30:00 0.0416667 27-OCT-2010 21:00:00 22214.9 3> 27-OCT-2010 22:30:00 0.0416667 27-OCT-2010 22:00:00 22214.94 {noformat} The problem seems to be that the 10m data begin at 20:11:17 before the 1m data at 21:08:43. The first write of the 1m data to the output netCDF file gives the file its shape with the KLIMITS= and LLIMITS= options. Writing the 10m data, which begins in before the bounds that were defined violates Ferret's axis ordering logic. To test this hypothesis edit the SAVE statement in the temporary .jnl file to make the start time for the 10m data 21:00, execute it and then examine the output file: {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 21:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc (Another ferret session) > ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 31-May-11 12:19 yes? use "/tmp/OS_MBARI-M1_20101027_R_TS.nc" yes? list/l=1:10 temp VARIABLE : Hourly sea_water_temperature (celsius) FILENAME : OS_MBARI-M1_20101027_R_TS.nc FILEPATH : /tmp/ SUBSET : 11 by 10 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 27-OCT-2010 21:30 / 1: 14.09 13.60 .... .... .... .... .... .... .... .... .... 27-OCT-2010 22:30 / 2: 14.17 13.62 .... .... .... .... .... .... .... .... .... 27-OCT-2010 23:30 / 3: 14.34 13.63 .... .... .... .... .... .... .... .... .... 28-OCT-2010 00:30 / 4: 14.28 13.69 .... .... .... .... .... .... .... .... .... 28-OCT-2010 01:30 / 5: 14.22 13.72 .... .... .... .... .... .... .... .... .... 28-OCT-2010 02:30 / 6: 14.21 13.78 .... .... .... .... .... .... .... .... .... 28-OCT-2010 03:30 / 7: 14.21 13.76 .... .... .... .... .... .... .... .... .... 28-OCT-2010 04:30 / 8: 14.09 13.79 .... .... .... .... .... .... .... .... .... 28-OCT-2010 05:30 / 9: 14.01 13.77 .... .... .... .... .... .... .... .... .... 28-OCT-2010 06:30 / 10: 13.89 13.76 .... .... .... .... .... .... .... .... .... yes? {noformat} O.K.\! That looks good. The 10m data got written. Now to fix it for production. This deployment was a special case where the 1m data start after the 10m data. Typically, it's most convenient to have all of the microcats have the same deployment start time. This makes the follow on data processing much simpler. All of the deployment start times in SSDS_METADTA for the M1 201010 deployment were adjusted to be the same as the start time for the Mooring deployment. Performing this step is now part of the [standard operating procedure for perming mooring turns|SSDS:OASIS Mooring turn]. Once SSDS_METADATA has been updated the individual instrument netCDF files must be created by DStoNetCDF.pl before combineTS.pl in run. There are still some problems with gappy data at 40m and below in the \_TS file. To debug these we'll make another copy of the OS_MBARI-M1_20101027_R_TS.jnl to /tmp and edit it to go through the processing of the 40m data saving the data to a netCDF file in /tmp so as not to disturb the production processing. We also comment out all of the "CANCEL DATA 1" statements so that we can examine data from the files. This file is executed and we see no errors. We need to examine the input data and the Ferret variables that are constructed for the gridding. Here are the commands from the .jnl that perform the the gridding for the 40m data: {noformat} ! ! Remove temperature outliers before gridding ! LET Temperature_QFLAG = IF Temperature GT 2 AND Temperature LT 20 THEN 1 ELSE (-99999) SET VAR/BAD=-99999 Temperature_QFLAG LET Temperature_QC = Temperature_QFLAG * Temperature ! ! Compute my own mean of the data, making sure to assign missing values in the gaps in the Gap FLAG ! Allow at least 1 data point in each destination cell - there are usually 6 for 10 minute data ! LET Temperature_GOOD = Temperature_QC[gt=TIME@SUM] / Temperature_QC[gt=TIME@NGD] LET Temperature_GFLAG = IF Temperature[gt=TIME@NGD,gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] LT 1 THEN (-99999) ELSE 1 SET VAR/BAD=-99999 Temperature_GFLAG LET TEMP = Temperature_GFLAG * Temperature_GOOD[gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] {noformat} Here are some sample Ferret commands for plotting these data from the session that just executed the OS_MBARI-M1_20101027_R_TS.jnl script: {noformat} yes? go "OS_MBARI-M1_20101027_R_TS.jnl" yes? set region/t=1-may-2011:1-jun-2011 yes? plot Temperature_QC yes? plot/ov/symbol=1 temperature yes? set win 2 yes? plot Temperature_QC[gt=TIME@NGD] yes? plot TEMPERATURE_QC[GT=TIME@SUM] {noformat} These last two plot commands indicate the source of the problem of the gappy data from the inductive modem microcats. Another big clue is the comment in the .jnl file referring to the 10-minute input data and having at least 6 good values within each gridding cell. As can be seen in the plot we often have less than 1 for the @NGD value for the 40m data. These data are hourly and we should be using a different gridding algorithm for them. !m1_ngd_sum.gif|thumbnail! The October 2010 M1 deployment was the first one where all the inductive modem connected CTDs were logged as separate instruments, and the first deployment where the serial microcat processing software was used to process the inductive modem microcat data. The fix for this problem appears to be to do a different gridding for the hourly data from 40m and below. This change was implemented in the code on 2 June 2011 using the Ferret same gridding transform (@MAX) that was previously used for the inductive modem data processing.  Here is the after figure: !wind_SEA_WATER_TEMPERATURE_HR_last30_after.gif|thumbnail! The subsurface data looks a lot better, but there are still some gaps.  If the gaps extend across the whole water column encompassing both the inductive modem and serial microcats then perhaps the gap is due to an interruption in data processing through oasisToSSDS.  This can be remedied by reprocessing the telemetered data by following the README instructions in /u/ssdsadmin/dev/DPforSSDS/oasis/scripts on elvis.   It is worthwhile to do another check on the quality of the gridding that is now being performed.  The following Ferret exercice will show the input data compared to the gridded data. {noformat} USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" SET REGION/T="29-May-2011 18:13":"02-Jun-2011 18:13" shade SEA_WATER_TEMPERATURE_HR set win 2 plot/k=1/vlimits=7:13 SEA_WATER_TEMPERATURE_HR repeat/k=2:11 plot/ov SEA_WATER_TEMPERATURE_HR {noformat} Produces these images showing missing data, especially from the 20m and 40m microcats: !m1_shade.gif|thumbnail!     \\]]> 17236104 62 64 55 57 8388776 8356015 56 58 8388778 8356017 58 60 8388771 8356010 8388773 8356012 8388774 8356013 77 79 8388829 8356070 76 78 75 77 8388827 8356068 82 channel=0, temp=48.51, vcc=3.2316, txBiasI=3.9796, txPower=0.4652, rxPower=0.0717, status=0x0 channel=3, temp=51.10, vcc=3.2252, txBiasI=4.4665, txPower=0.5313, rxPower=0.1337, status=0x0 }}} 1. Medusa ('''SSDS ID 1485'''): Looks like the previous (might have actually replaced that one because it has current data).{{{ -> channel=0, temp=51.69, vcc=3.2208, txBiasI=4.2839, txPower=-3.3320, rxPower=-10.6312, status=0x0 channel=3, temp=54.25, vcc=3.2142, txBiasI=4.7679, txPower=-2.7678, rxPower=-8.7681, status=0x0 }}} 1. Digital Camera 1. Thumbnail page of all images from the camera (click on image to get full image) 1. Sounds like images will come back as encapsulated JPG 1. CTD (SBE 16+) 1. Plot raw data like on CIMT 1. Process data to calculate salinity 1. Plot salinity along with raw 1. Current Meter, ADCP, RDI Sentinel 1. John R thought the processing would be very similar to the ADCP's we have seen on MTM and M0. 1. Current Meter, profiling, Aquadaopp HR Profiler, 6000m 1. This is a complicated vector plot that will be calculated by matlab script. Assume we will just display the image 1. Flourometer/Backscatter, Wetlabs ECO BB2F triplett 1. Unknown, but assuming this would be a similar process that Reiko is running for M0 1. Oxygen, Aanderaa 1. Unknown 1. Turbidity, Nephalometer, C-star, 6000m 1. Unknown 1. MSE Axis BIN Medusa subnode 2 ('''SSDS ID 1409'''): Again, just message packets. 1. MSP430 ('''SSDS ID 1413'''): Again, another MSP. Nothing new and has current data. 1. MBARI Medusa ('''SSDS ID 1442'''): No current data, but recent data looks like {{{ -> channel=3, temp=49.40, vcc=3.2308, txBiasI=2.6144, txPower=0.5348, rxPower=0.2863, status=0x0 }}} 1. Medusa ('''SSDS ID 1486'''):{{{ -> channel=3, temp=50.25, vcc=3.2476, txBiasI=2.6434, txPower=-2.6995, rxPower=-5.3036, status=0x0 }}} 1. Digital Camera 1. Thumbnail page of all images from the camera (click on image to get full image) 1. Sounds like images will come back as encapsulated JPG 1. CTD (SBE 16+) 1. Plot raw data like on CIMT 1. Process data to calculate salinity 1. Plot salinity along with raw 1. Current Meter, ADCP, RDI Sentinel 1. John R thought the processing would be very similar to the ADCP's we have seen on MTM and M0. 1. Current Meter, profiling, Aquadaopp HR Profiler, 6000m 1. This is a complicated vector plot that will be calculated by matlab script. Assume we will just display the image 1. Flourometer/Backscatter, Wetlabs ECO BB2F triplett 1. Unknown, but assuming this would be a similar process that Reiko is running for M0 1. Oxygen, Aanderaa 1. Unknown 1. Turbidity, Nephalometer, C-star, 6000m 1. Unknown 1. Vertical Profiler, McLane(FSI velocity, Seapoint OBS, Wetlabs FL, CT&P) 1. Unknown 1. Sediment Trap, Honjo 1. Unknown 1. Sediment Trap, IRS 1. Unknown 1. Outstanding Issues: 1. Instrument questions: 1. ASIMET LWR (John Ryan): Is there any post processing of the raw data and if so, what does it consist of? 1. ASIMET Radiometer, short-wave (John Ryan): Is there any post processing of the raw data and if so, what does it consist of? 1. ASIMET Relative Humidity and Temperature (Francisco Chavez): Is there any post processing of the raw data and if so, what does it consist of? 1. ASIMET Wind Speed Direction (Francisco Chavez): Is there any post processing of the raw data and if so, what does it consist of? 1. Power Can (Ed Mellinger): Any new features/graphs 1. Load Cell(Lance McBride): Any processing, any new plots? 1. Seahorse(Andy Hamilton): Any processing, feedback or data plots from seahorse? Any linked images? Do you want processing and products tracked? 1. MBARI Medusa(Mark Chaffey): Do we want plots of any of this data? Any post processing (and resubmit/track)? 1. ASIMET Barometric(Chavez or Ryan): Do plots of raw data make sense? Any post processing (and resubmit/track)? 1. All Satlantic Hyperspectral Radiometers (Chavez/Ryan): What/who is going to do the post processing. What does it look like and can we feedback/link to the results. 1. Digital Camera (Jim Barry, Mark Chaffey): is still an unknown as to what will be returned to SSDS. Mark sent request to Jim B for requirements from the camera and that will help answer the questions for SSDS. 1. CTD (SBE 16+) (Bill Ussler/Doug Conlin): Seabird software was identified as possibly doing the post processing, need to understand process in detail 1. ADCP RDI Sentinel (Coenen): What is the post processing for this instrument? Can we feedback/link data products? 1. Aquadopp (Ussler): Contact Leslie to get post processing software. 1. Wetlabs ECO BB2F Triplett (Ussler): Follow up to try and get example data to figure out what next steps are 1. Aanderaa Oxygen (Barry): Need to follow up to see what the data looks like and what post processing will be done. 1. Nephalomter (Ussler): Get details on post processing. Is this a SIAM instrument? 1. McLane Profiler(Hamilton): Get any details on post processing and if it is to link back to SSDS. 1. Sediment Traps(Chavez): What will the data look like, are we going to put that in SSDS? 1. General Issues: 1. No decision has been made on MBARI's data policy for this experiment, currently the SSDS team is assuming all data will be open. The public access to data issue was discussed and the next step is to have the scientist's meet to come up with idea of how MSE data should be accessed by the public (if at all) and then give that feedback to the MT for review/approval (note the current default is that data that goes into SSDS is public) 1. Documentation in spreadsheet incomplete -- we need to go to each of the reps and collect this data 1. Don't know driver writers for many nominally SIAM instruments 1. Not sure what the summary/snapshot data (ADCP, OCR, Nortek, Weblabs) will look like and if we are supposed to do anything with it. == Related Data Management Systems and Projects == 1. [http://www.openioos.org/index.html OpenIOOS.org] 1. [http://www.geongrid.org GEON] 1. [https://www.earthsystemgrid.org Earth Systems Grid] 1. [http://oceanexplorer.noaa.gov/ NOAA's OceanExplorer] 1. [http://www.gomoos.org/ GOMOOS] 1. [http://www.seamaven.org/sm.swf SeaMaven] 1. [http://seacoos.org/ SEACOOS] 1. [http://www.thecoolroom.org/ The COOL Room] 1. [http://earthobservations.org/ EARTH Observations] 1. [http://strategies.org/ Strategies.org] 1. [http://www.epic.noaa.gov/epic/software/dapper/ DAPPER (Look at DAPPER to serve SSDS data)] 1. [http://www.pacoos.org/ PACOOS] 1. [http://www.cencoos.org/ CENCOOS] 1. [http://egy.org/ EGY] 1. [http://www.oceansites.org/OceanSITES/ OceanSITES] 1. [http://ingrid.ldgo.columbia.edu/ LDGO] 1. Christoph Waldman (University of Bremen, Germany) (presented at MBARI on Jan 9, 2006) 1. Sensor interoperability and standardisation 1. ESONET I and ESONET II + ESONIM, KM2NET, ANTARES, ORION 1. He is working on standardized data quality and semantics 1. Quality Control: important to always state accuracy 1. Using SensorML to describe the sensor and data from sensor (but does not address semantics) ]]> 84 81 channel=0, temp=48.51, vcc=3.2316, txBiasI=3.9796, txPower=0.4652, rxPower=0.0717, status=0x0 channel=3, temp=51.10, vcc=3.2252, txBiasI=4.4665, txPower=0.5313, rxPower=0.1337, status=0x0 }}} 1. Medusa ('''SSDS ID 1485'''): Looks like the previous (might have actually replaced that one because it has current data).{{{ -> channel=0, temp=51.69, vcc=3.2208, txBiasI=4.2839, txPower=-3.3320, rxPower=-10.6312, status=0x0 channel=3, temp=54.25, vcc=3.2142, txBiasI=4.7679, txPower=-2.7678, rxPower=-8.7681, status=0x0 }}} 1. Digital Camera 1. Thumbnail page of all images from the camera (click on image to get full image) 1. Sounds like images will come back as encapsulated JPG 1. CTD (SBE 16+) 1. Plot raw data like on CIMT 1. Process data to calculate salinity 1. Plot salinity along with raw 1. Current Meter, ADCP, RDI Sentinel 1. John R thought the processing would be very similar to the ADCP's we have seen on MTM and M0. 1. Current Meter, profiling, Aquadaopp HR Profiler, 6000m 1. This is a complicated vector plot that will be calculated by matlab script. Assume we will just display the image 1. Flourometer/Backscatter, Wetlabs ECO BB2F triplett 1. Unknown, but assuming this would be a similar process that Reiko is running for M0 1. Oxygen, Aanderaa 1. Unknown 1. Turbidity, Nephalometer, C-star, 6000m 1. Unknown 1. MSE Axis BIN Medusa subnode 2 ('''SSDS ID 1409'''): Again, just message packets. 1. MSP430 ('''SSDS ID 1413'''): Again, another MSP. Nothing new and has current data. 1. MBARI Medusa ('''SSDS ID 1442'''): No current data, but recent data looks like {{{ -> channel=3, temp=49.40, vcc=3.2308, txBiasI=2.6144, txPower=0.5348, rxPower=0.2863, status=0x0 }}} 1. Medusa ('''SSDS ID 1486'''):{{{ -> channel=3, temp=50.25, vcc=3.2476, txBiasI=2.6434, txPower=-2.6995, rxPower=-5.3036, status=0x0 }}} 1. Digital Camera 1. Thumbnail page of all images from the camera (click on image to get full image) 1. Sounds like images will come back as encapsulated JPG 1. CTD (SBE 16+) 1. Plot raw data like on CIMT 1. Process data to calculate salinity 1. Plot salinity along with raw 1. Current Meter, ADCP, RDI Sentinel 1. John R thought the processing would be very similar to the ADCP's we have seen on MTM and M0. 1. Current Meter, profiling, Aquadaopp HR Profiler, 6000m 1. This is a complicated vector plot that will be calculated by matlab script. Assume we will just display the image 1. Flourometer/Backscatter, Wetlabs ECO BB2F triplett 1. Unknown, but assuming this would be a similar process that Reiko is running for M0 1. Oxygen, Aanderaa 1. Unknown 1. Turbidity, Nephalometer, C-star, 6000m 1. Unknown 1. Vertical Profiler, McLane(FSI velocity, Seapoint OBS, Wetlabs FL, CT&P) 1. Unknown 1. Sediment Trap, Honjo 1. Unknown 1. Sediment Trap, IRS 1. Unknown 1. Outstanding Issues: 1. Instrument questions: 1. ASIMET LWR (John Ryan): Is there any post processing of the raw data and if so, what does it consist of? 1. ASIMET Radiometer, short-wave (John Ryan): Is there any post processing of the raw data and if so, what does it consist of? 1. ASIMET Relative Humidity and Temperature (Francisco Chavez): Is there any post processing of the raw data and if so, what does it consist of? 1. ASIMET Wind Speed Direction (Francisco Chavez): Is there any post processing of the raw data and if so, what does it consist of? 1. Power Can (Ed Mellinger): Any new features/graphs 1. Load Cell(Lance McBride): Any processing, any new plots? 1. Seahorse(Andy Hamilton): Any processing, feedback or data plots from seahorse? Any linked images? Do you want processing and products tracked? 1. MBARI Medusa(Mark Chaffey): Do we want plots of any of this data? Any post processing (and resubmit/track)? 1. ASIMET Barometric(Chavez or Ryan): Do plots of raw data make sense? Any post processing (and resubmit/track)? 1. All Satlantic Hyperspectral Radiometers (Chavez/Ryan): What/who is going to do the post processing. What does it look like and can we feedback/link to the results. 1. Digital Camera (Jim Barry, Mark Chaffey): is still an unknown as to what will be returned to SSDS. Mark sent request to Jim B for requirements from the camera and that will help answer the questions for SSDS. 1. CTD (SBE 16+) (Bill Ussler/Doug Conlin): Seabird software was identified as possibly doing the post processing, need to understand process in detail 1. ADCP RDI Sentinel (Coenen): What is the post processing for this instrument? Can we feedback/link data products? 1. Aquadopp (Ussler): Contact Leslie to get post processing software. 1. Wetlabs ECO BB2F Triplett (Ussler): Follow up to try and get example data to figure out what next steps are 1. Aanderaa Oxygen (Barry): Need to follow up to see what the data looks like and what post processing will be done. 1. Nephalomter (Ussler): Get details on post processing. Is this a SIAM instrument? 1. McLane Profiler(Hamilton): Get any details on post processing and if it is to link back to SSDS. 1. Sediment Traps(Chavez): What will the data look like, are we going to put that in SSDS? 1. General Issues: 1. No decision has been made on MBARI's data policy for this experiment, currently the SSDS team is assuming all data will be open. The public access to data issue was discussed and the next step is to have the scientist's meet to come up with idea of how MSE data should be accessed by the public (if at all) and then give that feedback to the MT for review/approval (note the current default is that data that goes into SSDS is public) 1. Documentation in spreadsheet incomplete -- we need to go to each of the reps and collect this data 1. Don't know driver writers for many nominally SIAM instruments 1. Not sure what the summary/snapshot data (ADCP, OCR, Nortek, Weblabs) will look like and if we are supposed to do anything with it. == Related Data Management Systems and Projects == 1. [http://www.openioos.org/index.html OpenIOOS.org] 1. [http://www.geongrid.org GEON] 1. [https://www.earthsystemgrid.org Earth Systems Grid] 1. [http://oceanexplorer.noaa.gov/ NOAA's OceanExplorer] 1. [http://www.gomoos.org/ GOMOOS] 1. [http://www.seamaven.org/sm.swf SeaMaven] 1. [http://seacoos.org/ SEACOOS] 1. [http://www.thecoolroom.org/ The COOL Room] 1. [http://earthobservations.org/ EARTH Observations] 1. [http://strategies.org/ Strategies.org] 1. [http://www.epic.noaa.gov/epic/software/dapper/ DAPPER (Look at DAPPER to serve SSDS data)] 1. [http://www.pacoos.org/ PACOOS] 1. [http://www.cencoos.org/ CENCOOS] 1. [http://egy.org/ EGY] 1. [http://www.oceansites.org/OceanSITES/ OceanSITES] 1. [http://ingrid.ldgo.columbia.edu/ LDGO] 1. Christoph Waldman (University of Bremen, Germany) (presented at MBARI on Jan 9, 2006) 1. Sensor interoperability and standardisation 1. ESONET I and ESONET II + ESONIM, KM2NET, ANTARES, ORION 1. He is working on standardized data quality and semantics 1. Quality Control: important to always state accuracy 1. Using SensorML to describe the sensor and data from sensor (but does not address semantics) ]]> 83 8388833 8356076 8388831 8356072 8388821 8356062 8388819 8356060 74 76 73 75 8388824 8356065 8388823 8356064 11273152 11240415 8388814 8356055 11273154 11240417 91 93 92 94 8388812 8356053 3670860 15 {noformat} to {noformat} 1 {noformat} Which should effectively make the RuminateMDB a singleton which should alleviate our deadlock issues that we were having (at least at Ruminate step). {note}]]> 3638121 11273149 login result page /loginResult.jsp Engineering Distribution List Research Distribution List ITD Distribution List DMO Distribution List OED Distribution List {noformat} # Whew, once all that is done, you are ready to build and deploy the SSDS. Open command prompt, cd to directory where SSDS was checked out and type {noformat} ant -Dtarget=deploy {noformat} # Once the build is complete, open a terminal window, change directory to where the JBoss home is, and start JBoss {noformat} sudo ./bin/run.sh {noformat} # Once JBoss finishes startup, you should see the line: {noformat} 17:12:41,607 INFO [Server] JBoss (MX MicroKernel) [4.2.2.GA (build: SVNTag=JBoss_4_2_2_GA date=200710221139)] Started in 19s:45ms {noformat} and hopefully you did not see any stack traces fly by during startup. If all looks OK, you should be able to go the SSDS_Metadata database and see newly created tables that will be where SSDS will store is metadata. You can also open a browser and go to [http://localhost:8080/ssds-docs/] to see a simple page with some links to SSDS documentation and various utilities and libraries. {note:title=Still to document} # Configure mod_jk in Apache/JBoss # Configure SSL for login.jsp page {note} h3. Setup an Eclipse project: In order to actually do some development on the SSDS code base, you can use an IDE like Eclipse to edit the code and use ant to deploy your changes to JBoss. To setup an Eclipse project. # Install eclipse that you download from [http://www.eclipse.org] # Startup Eclipse and select File->New->Java Project. # In the New Project Window, select 'Create project from existing source' and browse to where you checked out the SSDS source code. Once you configure the project correctly, you should have the following project settings: ## There should be two source directories: {noformat} src/java {noformat} and {noformat} src/gen {noformat} (src/gen is created by ant during build) # Add all jars in the lib directory # Add src/resources/build/antlr/antlr-2.7.5.jar # Add JBoss jars ## JBOSS_HOME/client/activation.jar ## JBOSS_HOME/client/servlet-api.jar ## JBOSS_HOME/client/jboss-j2ee.jar ## JBOSS_HOME/client/log4j.jar ## JBOSS_HOME/server/default/lib/commons-codec.jar ## JBOSS_HOME/server/default/lib/commons-collections.jar ## JBOSS_HOME/server/default/lib/commons-httpclient.jar ## JBOSS_HOME/server/default/lib/hibernate3.jar ## JBOSS_HOME/server/default/lib/mail.jar # output set to build/classes (to align with ant's build files) h3. Create a FlexBuilder (plug-in) project: # Start Eclipse with Flex-Builder plug-in installed # File->New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/build/flex and choose it for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). {note:title=Will create a Main.mxml} Note that the creation of the Flex Builder project will create a Main.mxml file. To fix this, right click on explorer.mxml and choose 'Set As Default Application', then you can delete the Main.mxml file. {note} # Click on Finish NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 11240412 85 87 86 88 3670855 3638116 83 channel=0, temp=48.51, vcc=3.2316, txBiasI=3.9796, txPower=0.4652, rxPower=0.0717, status=0x0 channel=3, temp=51.10, vcc=3.2252, txBiasI=4.4665, txPower=0.5313, rxPower=0.1337, status=0x0}} ### Medusa (*SSDS ID 1485*): Looks like the previous (might have actually replaced that one because it has current data). {{-> channel=0, temp=51.69, vcc=3.2208, txBiasI=4.2839, txPower=-3.3320, rxPower=-10.6312, status=0x0 channel=3, temp=54.25, vcc=3.2142, txBiasI=4.7679, txPower=-2.7678, rxPower=-8.7681, status=0x0}} ### Digital Camera #### Thumbnail page of all images from the camera (click on image to get full image) #### Sounds like images will come back as encapsulated JPG ## CTD (SBE 16+) #### Plot raw data like on CIMT 1. Process data to calculate salinity 1. Plot salinity along with raw 1. Current Meter, ADCP, RDI Sentinel 1. John R thought the processing would be very similar to the ADCP's we have seen on MTM and M0. 1. Current Meter, profiling, Aquadaopp HR Profiler, 6000m 1. This is a complicated vector plot that will be calculated by matlab script. Assume we will just display the image 1. Flourometer/Backscatter, Wetlabs ECO BB2F triplett 1. Unknown, but assuming this would be a similar process that Reiko is running for M0 1. Oxygen, Aanderaa 1. Unknown 1. Turbidity, Nephalometer, C-star, 6000m 1. Unknown 1. MSE Axis BIN Medusa subnode 2 ('''SSDS ID 1409'''): Again, just message packets. 1. MSP430 ('''SSDS ID 1413'''): Again, another MSP. Nothing new and has current data. 1. MBARI Medusa ('''SSDS ID 1442'''): No current data, but recent data looks like {{{ -> channel=3, temp=49.40, vcc=3.2308, txBiasI=2.6144, txPower=0.5348, rxPower=0.2863, status=0x0 }}} 1. Medusa ('''SSDS ID 1486'''):{{{ -> channel=3, temp=50.25, vcc=3.2476, txBiasI=2.6434, txPower=-2.6995, rxPower=-5.3036, status=0x0 }}} 1. Digital Camera 1. Thumbnail page of all images from the camera (click on image to get full image) 1. Sounds like images will come back as encapsulated JPG 1. CTD (SBE 16+) 1. Plot raw data like on CIMT 1. Process data to calculate salinity 1. Plot salinity along with raw 1. Current Meter, ADCP, RDI Sentinel 1. John R thought the processing would be very similar to the ADCP's we have seen on MTM and M0. 1. Current Meter, profiling, Aquadaopp HR Profiler, 6000m 1. This is a complicated vector plot that will be calculated by matlab script. Assume we will just display the image 1. Flourometer/Backscatter, Wetlabs ECO BB2F triplett 1. Unknown, but assuming this would be a similar process that Reiko is running for M0 1. Oxygen, Aanderaa 1. Unknown 1. Turbidity, Nephalometer, C-star, 6000m 1. Unknown 1. Vertical Profiler, McLane(FSI velocity, Seapoint OBS, Wetlabs FL, CT&P) 1. Unknown 1. Sediment Trap, Honjo 1. Unknown 1. Sediment Trap, IRS 1. Unknown 1. Outstanding Issues: 1. Instrument questions: 1. ASIMET LWR (John Ryan): Is there any post processing of the raw data and if so, what does it consist of? 1. ASIMET Radiometer, short-wave (John Ryan): Is there any post processing of the raw data and if so, what does it consist of? 1. ASIMET Relative Humidity and Temperature (Francisco Chavez): Is there any post processing of the raw data and if so, what does it consist of? 1. ASIMET Wind Speed Direction (Francisco Chavez): Is there any post processing of the raw data and if so, what does it consist of? 1. Power Can (Ed Mellinger): Any new features/graphs 1. Load Cell(Lance McBride): Any processing, any new plots? 1. Seahorse(Andy Hamilton): Any processing, feedback or data plots from seahorse? Any linked images? Do you want processing and products tracked? 1. MBARI Medusa(Mark Chaffey): Do we want plots of any of this data? Any post processing (and resubmit/track)? 1. ASIMET Barometric(Chavez or Ryan): Do plots of raw data make sense? Any post processing (and resubmit/track)? 1. All Satlantic Hyperspectral Radiometers (Chavez/Ryan): What/who is going to do the post processing. What does it look like and can we feedback/link to the results. 1. Digital Camera (Jim Barry, Mark Chaffey): is still an unknown as to what will be returned to SSDS. Mark sent request to Jim B for requirements from the camera and that will help answer the questions for SSDS. 1. CTD (SBE 16+) (Bill Ussler/Doug Conlin): Seabird software was identified as possibly doing the post processing, need to understand process in detail 1. ADCP RDI Sentinel (Coenen): What is the post processing for this instrument? Can we feedback/link data products? 1. Aquadopp (Ussler): Contact Leslie to get post processing software. 1. Wetlabs ECO BB2F Triplett (Ussler): Follow up to try and get example data to figure out what next steps are 1. Aanderaa Oxygen (Barry): Need to follow up to see what the data looks like and what post processing will be done. 1. Nephalomter (Ussler): Get details on post processing. Is this a SIAM instrument? 1. McLane Profiler(Hamilton): Get any details on post processing and if it is to link back to SSDS. 1. Sediment Traps(Chavez): What will the data look like, are we going to put that in SSDS? 1. General Issues: 1. No decision has been made on MBARI's data policy for this experiment, currently the SSDS team is assuming all data will be open. The public access to data issue was discussed and the next step is to have the scientist's meet to come up with idea of how MSE data should be accessed by the public (if at all) and then give that feedback to the MT for review/approval (note the current default is that data that goes into SSDS is public) 1. Documentation in spreadsheet incomplete -- we need to go to each of the reps and collect this data 1. Don't know driver writers for many nominally SIAM instruments 1. Not sure what the summary/snapshot data (ADCP, OCR, Nortek, Weblabs) will look like and if we are supposed to do anything with it. == Related Data Management Systems and Projects == 1. [http://www.openioos.org/index.html OpenIOOS.org] 1. [http://www.geongrid.org GEON] 1. [https://www.earthsystemgrid.org Earth Systems Grid] 1. [http://oceanexplorer.noaa.gov/ NOAA's OceanExplorer] 1. [http://www.gomoos.org/ GOMOOS] 1. [http://www.seamaven.org/sm.swf SeaMaven] 1. [http://seacoos.org/ SEACOOS] 1. [http://www.thecoolroom.org/ The COOL Room] 1. [http://earthobservations.org/ EARTH Observations] 1. [http://strategies.org/ Strategies.org] 1. [http://www.epic.noaa.gov/epic/software/dapper/ DAPPER (Look at DAPPER to serve SSDS data)] 1. [http://www.pacoos.org/ PACOOS] 1. [http://www.cencoos.org/ CENCOOS] 1. [http://egy.org/ EGY] 1. [http://www.oceansites.org/OceanSITES/ OceanSITES] 1. [http://ingrid.ldgo.columbia.edu/ LDGO] 1. Christoph Waldman (University of Bremen, Germany) (presented at MBARI on Jan 9, 2006) 1. Sensor interoperability and standardisation 1. ESONET I and ESONET II + ESONIM, KM2NET, ANTARES, ORION 1. He is working on standardized data quality and semantics 1. Quality Control: important to always state accuracy 1. Using SensorML to describe the sensor and data from sensor (but does not address semantics) ]]> 85 3670858 15 {noformat} to {noformat} 1 {noformat} Which should effectively make the RuminateMDB a singleton which should alleviate our deadlock issues that we were having (at least at Ruminate step). {note}]]> 3638119 84 86 3670857 3638118 90 92 87 89 88 90 3670853 3638114 17268933 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat}\\ h4. B. Tips for debugging with Ferret To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the change log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is [http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log] # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: [http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED] To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:`PSAL,return=lend`/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC !-> LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:5182/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the first 3 temperature values from the 1m and 10m input CTD data and the first 3 times of the output axis: {noformat} yes? list/l=1:3/d=1 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1338 at original sampling intervals FILENAME : m1_ctd0001_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 1 122W 1 27-OCT-2010 21:08:43 / 1: 14.01 27-OCT-2010 21:22:27 / 2: 14.10 27-OCT-2010 21:28:23 / 3: 14.10 yes? list/l=1:3/d=2 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 27-OCT-2010 20:11:17 / 1: 13.38 27-OCT-2010 20:21:16 / 2: 13.59 27-OCT-2010 20:31:16 / 3: 13.45 yes? show/l=1:3 axis time name axis # pts start end TIME TIME 5181 r 27-OCT-2010 20:30 31-MAY-2011 16:30 T0 = 01-JAN-1950 00:00:00 Axis span (to cell edges) = 215.875 L T TBOX TBOXLO TSTEP (DAYS) 1> 27-OCT-2010 20:30:00 0.0416667 27-OCT-2010 20:00:00 22214.85 2> 27-OCT-2010 21:30:00 0.0416667 27-OCT-2010 21:00:00 22214.9 3> 27-OCT-2010 22:30:00 0.0416667 27-OCT-2010 22:00:00 22214.94 {noformat}\\ h4. C. Fixing the start times for all the child instrument deployments The problem seems to be that the 10m data begin at 20:11:17 before the 1m data at 21:08:43. The first write of the 1m data to the output netCDF file gives the file its shape with the KLIMITS= and LLIMITS= options. Writing the 10m data, which begins in before the bounds that were defined violates Ferret's axis ordering logic. To test this hypothesis edit the SAVE statement in the temporary .jnl file to make the start time for the 10m data 21:00, execute it and then examine the output file: {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 21:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc (Another ferret session) > ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 31-May-11 12:19 yes? use "/tmp/OS_MBARI-M1_20101027_R_TS.nc" yes? list/l=1:10 temp VARIABLE : Hourly sea_water_temperature (celsius) FILENAME : OS_MBARI-M1_20101027_R_TS.nc FILEPATH : /tmp/ SUBSET : 11 by 10 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 27-OCT-2010 21:30 / 1: 14.09 13.60 .... .... .... .... .... .... .... .... .... 27-OCT-2010 22:30 / 2: 14.17 13.62 .... .... .... .... .... .... .... .... .... 27-OCT-2010 23:30 / 3: 14.34 13.63 .... .... .... .... .... .... .... .... .... 28-OCT-2010 00:30 / 4: 14.28 13.69 .... .... .... .... .... .... .... .... .... 28-OCT-2010 01:30 / 5: 14.22 13.72 .... .... .... .... .... .... .... .... .... 28-OCT-2010 02:30 / 6: 14.21 13.78 .... .... .... .... .... .... .... .... .... 28-OCT-2010 03:30 / 7: 14.21 13.76 .... .... .... .... .... .... .... .... .... 28-OCT-2010 04:30 / 8: 14.09 13.79 .... .... .... .... .... .... .... .... .... 28-OCT-2010 05:30 / 9: 14.01 13.77 .... .... .... .... .... .... .... .... .... 28-OCT-2010 06:30 / 10: 13.89 13.76 .... .... .... .... .... .... .... .... .... yes? {noformat} O.K.\! That looks good. The 10m data got written. Now to fix it for production. This deployment was a special case where the 1m data start after the 10m data. Typically, it's most convenient to have all of the microcats have the same deployment start time. This makes the follow on data processing much simpler. All of the deployment start times in SSDS_METADTA for the M1 201010 deployment were adjusted to be the same as the start time for the Mooring deployment. Performing this step is now part of the [standard operating procedure for perming mooring turns|SSDS:OASIS Mooring turn]. Once SSDS_METADATA has been updated the individual instrument netCDF files must be created by DStoNetCDF.pl before combineTS.pl in run. h4. D. Examining the gridding method There are still some problems with gappy data at 40m and below in the \_TS file. To debug these we'll make another copy of the OS_MBARI-M1_20101027_R_TS.jnl to /tmp and edit it to go through the processing of the 40m data saving the data to a netCDF file in /tmp so as not to disturb the production processing. We also comment out all of the "CANCEL DATA 1" statements so that we can examine data from the files. This file is executed and we see no errors. We need to examine the input data and the Ferret variables that are constructed for the gridding. Here are the commands from the .jnl that perform the the gridding for the 40m data: {noformat} ! ! Remove temperature outliers before gridding ! LET Temperature_QFLAG = IF Temperature GT 2 AND Temperature LT 20 THEN 1 ELSE (-99999) SET VAR/BAD=-99999 Temperature_QFLAG LET Temperature_QC = Temperature_QFLAG * Temperature ! ! Compute my own mean of the data, making sure to assign missing values in the gaps in the Gap FLAG ! Allow at least 1 data point in each destination cell - there are usually 6 for 10 minute data ! LET Temperature_GOOD = Temperature_QC[gt=TIME@SUM] / Temperature_QC[gt=TIME@NGD] LET Temperature_GFLAG = IF Temperature[gt=TIME@NGD,gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] LT 1 THEN (-99999) ELSE 1 SET VAR/BAD=-99999 Temperature_GFLAG LET TEMP = Temperature_GFLAG * Temperature_GOOD[gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] {noformat} Here are some sample Ferret commands for plotting these data from the session that just executed the OS_MBARI-M1_20101027_R_TS.jnl script: {noformat} yes? go "OS_MBARI-M1_20101027_R_TS.jnl" yes? set region/t=1-may-2011:1-jun-2011 yes? plot Temperature_QC yes? plot/ov/symbol=1 temperature yes? set win 2 yes? plot Temperature_QC[gt=TIME@NGD] yes? plot TEMPERATURE_QC[GT=TIME@SUM] {noformat} These last two plot commands indicate the source of the problem of the gappy data from the inductive modem microcats. Another big clue is the comment in the .jnl file referring to the 10-minute input data and having at least 6 good values within each gridding cell. As can be seen in the plot we often have less than 1 for the @NGD value for the 40m data. These data are hourly and we should be using a different gridding algorithm for them. !m1_ngd_sum.gif|thumbnail! The October 2010 M1 deployment was the first one where all the inductive modem connected CTDs were logged as separate instruments, and the first deployment where the serial microcat processing software was used to process the inductive modem microcat data. The fix for this problem appears to be to do a different gridding for the hourly data from 40m and below. This change was implemented in the code on 2 June 2011 using the Ferret same gridding transform (@MAX) that was previously used for the inductive modem data processing.  Here is the after figure: !wind_SEA_WATER_TEMPERATURE_HR_last30_after.gif|thumbnail! The subsurface data looks a lot better, but there are still some gaps.  If the gaps extend across the whole water column encompassing both the inductive modem and serial microcats then perhaps the gap is due to an interruption in data processing through oasisToSSDS.  This can be remedied by reprocessing the telemetered data by following the README instructions in /u/ssdsadmin/dev/DPforSSDS/oasis/scripts on elvis. h4. E. Testing the results of the gridding It is worthwhile to do another check on the quality of the gridding that is now being performed.  The following Ferret exercice will show the input data compared to the gridded data. {noformat} USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" SET REGION/T="29-May-2011 18:13":"02-Jun-2011 18:13" shade SEA_WATER_TEMPERATURE_HR set win 2 plot/k=1/vlimits=7:13 SEA_WATER_TEMPERATURE_HR repeat/k=2:11 plot/ov SEA_WATER_TEMPERATURE_HR {noformat} Produces these images showing missing data, especially from the 20m and 40m microcats: !m1_shade.gif|thumbnail! !m1_lines.gif|thumbnail! Let's compare the original instruments date from 20m and 40m to the gridded product, using the same Ferret session from above:" {noformat} use "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0020_20101027_original.nc" plot/symbol=1 temperature[d=2] plot/ov SEA_WATER_TEMPERATURE_HR[k=3,d=1] use "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0040_20101027_original.nc" plot/symbol=1 temperature[d=3] plot/ov SEA_WATER_TEMPERATURE_HR[k=4,d=1] {noformat} Gives: !m1_compare_20.gif|thumbnail! !m1_compare_40.gif|thumbnail! We obviously still have some gridding issues... h4. F. Rinse and repeat Now is the time to iterate using our method of starting with the .jnl file that aggregates and grids the data to the OceanSITES \_TS.nc file going back to step D. \\]]> 17236169 8388843 8356086 8388844 8356087 8388845 8356088 8388835 8356078 8388837 8356080 8388839 8356082 8388841 8356084 16449621 metadataRef # metadataRef->metadataSequenceNumber # metadataRef->dataDescriptionID | # metadataRef # metadataSequenceNumber # dataDescriptionID | | parentId | direct copy: # parentId->parentId # parentId->platformID | # parentId # platformID | | recordType | # MetadataPacket: recordType to 0 # Other: direct copy | recordType | | | # If MetadataPacket, packetType = 0 # If SensorDataPacket, packetType = 1 # If DeviceMessagePacket, packetType = 2 | packetType | | firstBufferLength | ignored | | | firstBuffer | First buffer depends on which type of packet # If MetadataPacket, copy "bytes" buffer # If SensorDataPacket, copy "dataBuffer" # If DeviceMessagePacket, copy "message" | firstBuffer | | secondBufferLength | ignored | | | secondBuffer | Only exists if MetadataPacket and will copy over "cause" buffer | secondBuffer | h6. DevicePacket to SIAM Byte Array This is done by the SIAM Exportable Packet class || DevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataRef | direct copy | metadataRef | | parentId | direct copy | parentId | | recordType | direct copy | recordType | | | This is set based on what type of packet: # If MetadataPacket, set to 0x0101 # If SensorDataPacket, set to 0x0102 # If DeviceMessagePacket, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | firstBufferLength | direct copy (see note for first buffer) | firstBufferLength | | firstBuffer | This depends on the type of packet: # If MetadataPacket, "cause" bytes are copied over # If SensorDataPacket, "dataMessage" bytes are copied over # If DeviceMessagePacket, "message" bytes are copied over | firstBuffer | | secondBufferLength | only valid with MetadataPacket, but is copied directly over | secondBufferLength | | secondBuffer | only valid with MetadataPacket, and the "buffer" bytes are copied over | secondBuffer | h6. SSDSDevicePacket to SIAM Byte Array *Originally done in TransmogrifyMDB by calling SSDSDevicePacket.convertToPublishableVersion3ByteArray before passing byte array to method to translate to SSDS format and then send to ingest* || SSDSDevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataSequenceNumber | direct copy | metadataRef | | parentID | direct copy | parentID | | recordType | # If packetType = 0, set recordType = 0 # If packetType = 1, set recordType = recordType # If packetType = 2, set recordType = recordType | recordType | | | This is set based on what type of packet: # If packetType = 0, set to 0x0101 # If packetType = 1, set to 0x0102 # If packetType = 2, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | | This depends on the type of packet: # If packetType = 0, set to length of "otherBuffer" # If packetType = 1, set to length of "dataBuffer" # If packetType = 2, set to length of "dataBuffer" | firstBufferLength | | other/dataBuffer | This depends on the type of packet: # If packetType = 0, set to bytes from "otherBuffer" # If packetType = 1, set to bytes from "dataBuffer" # If packetType = 2, set to bytes from "dataBuffer" | firstBuffer | | | This only exists if it is packetType = 0 then it is set to the length of the "dataBuffer" | secondBufferLength | | dataBuffer | This only exists if it is packetType = 0 then it is set to the byte from the "dataBuffer" | secondBuffer | h5. SSDSDevicePacket to SSDS Byte Array *Originally done in SSDSDevicePacket.convertToVersion3ByteArray* || SSDSDevicePacket || Translation Rule || SSDS Byte Array || | sourceID | direct copy | sourceID | | systemTime | ignored during the translation directly, but used through getter methods for seconds and nanoseconds | | | timestampSeconds | direct copy (note that this is _sort of_ a direct copy, there are getter methods on SSDSDevicePacket that convert the systemTime to seconds and nanoseconds when called). | timestampSeconds | | timestampNanoseconds | direct copy (note that this is _sort of_ a direct copy, there are getter methods on SSDSDevicePacket that convert the systemTime to seconds and nanoseconds when called). | timestampNanoseconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | ignored | | | parentID | ignored | | | recordType | If packetType = 0, set packetSubType to 0, otherwise set to recordType | packetSubType | | packetType | Depends on packetType: # If packetType = 0, set to 1 # If packetType = 1, set to 0 # If packetType = 2, set to 4 | packetType | | metadataSequenceNumber | direct copy | metadataSequenceNumber | | dataDescriptionVersion | direct copy | dataDescriptionVersion | | platformID | direct copy | parentID | | | copy length of dataBuffer | firstBufferLength | | dataBuffer | direct copy | firstBuffer | | | copy length of otherBuffer | secondBufferLength | | otherBuffer | direct copy | secondBuffer | h6. SIAM Byte Array to SSDS Byte Array *Originally done in TransmogrifyMDB in checkAndPublishBytes method* || SIAM Byte Array || Translation Rules || SSDS Byte Array || | EX_DEVICEPACKET | ignored | | | serialVersionUID | ignored | | | sourceID | direct copy | sourceID | | | Depending on EX_XXXXXXXPACKET: # If MetadataPacket, set packetType to 1 # If SensorDataPacket, set packetType to 0 # If DeviceMessagePacket, set packetType to 4 | packetType | | | This was set using the SIAMMetadataTracker that tried to keep track of real version numbers based on XML in payload | metadataSequencNumber | | systemTime | Split into timestampSeconds and timestampNanoseconds | # timestampSeconds # timestampNanoSeconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | direct copy | dataDescriptionVersion | | parentId | direct copy | parentID | | recordType | If MetadataPacket (determined from EX_XXXXXXXPACKET), recordType set to 0, otherwise set to recordType | packetSubType | | EX_XXXXXXXPACKET | ignored in storage, but used in logic | | | serialVersionUID | ignored | | | first/secondBufferLength | If MetadataPacket (determined from EX_XXXXXXXPACKET), firstBufferLength is set to secondBufferLength so we can flip the "cause" and "buffer" bytes because it just made more sense since the cause was rarely populated. Otherwise set to firstBufferLength | firstBufferLength | | first/secondBuffer | If MetadataPacket (determined from EX_XXXXXXXPACKET), firstBuffer is set to secondBuffer so we can flip the "cause" and "buffer" bytes because it just made more sense since the cause was rarely populated. Otherwise set to firstBuffer | firstBuffer | | firstBufferLength | If MetadataPacket (determined from EX_XXXXXXXPACKET), secondBufferLength is set to firstBufferLength to flip "cause" and "buffer" bytes | secondBufferLength | | firstBuffer | If MetadataPacket (determined from EX_XXXXXXXPACKET), secondBuffer is set to firstBuffer to flip "cause" and "buffer" bytes | secondBuffer | h5. Exploration of Upgrade of Ingest/Transmogrify to AMQP In an effort to allow non-Java clients to send data to SSDS in the form of messages and to upgrade the messaging system to a technology that is more scalable and higher performance, an investigation of AMQP implementations was done. # [Qpid Exploration] ]]> 16416858 10388623 Databases->SSDS_Data->Tables and you should see something like this: !Figure 1.jpg|align=centre!]]> 10355897 17268800 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! (lots of other output skipped) SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the first 3 temperature values from the 1m and 10m input CTD data and the first 3 times of the output axis: {noformat} yes? list/l=1:3/d=1 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1338 at original sampling intervals FILENAME : m1_ctd0001_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 1 122W 1 27-OCT-2010 21:08:43 / 1: 14.01 27-OCT-2010 21:22:27 / 2: 14.10 27-OCT-2010 21:28:23 / 3: 14.10 yes? list/l=1:3/d=2 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 27-OCT-2010 20:11:17 / 1: 13.38 27-OCT-2010 20:21:16 / 2: 13.59 27-OCT-2010 20:31:16 / 3: 13.45 yes? show/l=1:3 axis time name axis # pts start end TIME TIME 5181 r 27-OCT-2010 20:30 31-MAY-2011 16:30 T0 = 01-JAN-1950 00:00:00 Axis span (to cell edges) = 215.875 L T TBOX TBOXLO TSTEP (DAYS) 1> 27-OCT-2010 20:30:00 0.0416667 27-OCT-2010 20:00:00 22214.85 2> 27-OCT-2010 21:30:00 0.0416667 27-OCT-2010 21:00:00 22214.9 3> 27-OCT-2010 22:30:00 0.0416667 27-OCT-2010 22:00:00 22214.94 {noformat} ]]> 17236034 10388624 Databases->SSDS_Data->Tables and you should see something like this: !Figure 1.jpg|align=centre! You are looking for the *DeviceQCPlotConfig* table and if you scroll down and right-click on it, then choose 'Open Table->Return all Rows' you will see the listing of all the configurations for creating plots (Figure 2). ]]> 10355898 17268801 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:`PSAL,return=lend`/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC !-> LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:5182/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the first 3 temperature values from the 1m and 10m input CTD data and the first 3 times of the output axis: {noformat} yes? list/l=1:3/d=1 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1338 at original sampling intervals FILENAME : m1_ctd0001_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 1 122W 1 27-OCT-2010 21:08:43 / 1: 14.01 27-OCT-2010 21:22:27 / 2: 14.10 27-OCT-2010 21:28:23 / 3: 14.10 yes? list/l=1:3/d=2 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 27-OCT-2010 20:11:17 / 1: 13.38 27-OCT-2010 20:21:16 / 2: 13.59 27-OCT-2010 20:31:16 / 3: 13.45 yes? show/l=1:3 axis time name axis # pts start end TIME TIME 5181 r 27-OCT-2010 20:30 31-MAY-2011 16:30 T0 = 01-JAN-1950 00:00:00 Axis span (to cell edges) = 215.875 L T TBOX TBOXLO TSTEP (DAYS) 1> 27-OCT-2010 20:30:00 0.0416667 27-OCT-2010 20:00:00 22214.85 2> 27-OCT-2010 21:30:00 0.0416667 27-OCT-2010 21:00:00 22214.9 3> 27-OCT-2010 22:30:00 0.0416667 27-OCT-2010 22:00:00 22214.94 {noformat} The problem seems to be that the 10m data begin at 20:11:17 before the 1m data at 21:08:43. The first write of the 1m data to the output netCDF file gives the file its shape with the KLIMITS= and LLIMITS= options. Writing the 10m data, which begins in before the bounds that were defined violates Ferret's axis ordering logic.]]> 17236035 8388631 8355864 10388626 Databases->SSDS_Data->Tables and you should see something like this: !Figure 1.jpg|align=centre! You are looking for the *DeviceQCPlotConfig* table and if you scroll down and right-click on it, then choose 'Open Table->Return all Rows' you will see the listing of all the configurations for creating plots (Figure 2). !Figure 2.jpg|align=centre! ]]> 10355900 5865699 5832941 17268796 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data # Executed the temporary truncated file from a Ferret session {noformat} {noformat} ]]> 17236030 10388620 10355894 16449623 metadataRef # metadataRef->metadataSequenceNumber # metadataRef->dataDescriptionID | # metadataRef # metadataSequenceNumber # dataDescriptionID | | parentId | direct copy: # parentId->parentId # parentId->platformID | # parentId # platformID | | recordType | # MetadataPacket: recordType to 0 # Other: direct copy | recordType | | | # If MetadataPacket, packetType = 0 # If SensorDataPacket, packetType = 1 # If DeviceMessagePacket, packetType = 2 | packetType | | firstBufferLength | ignored | | | firstBuffer | First buffer depends on which type of packet # If MetadataPacket, copy "bytes" buffer # If SensorDataPacket, copy "dataBuffer" # If DeviceMessagePacket, copy "message" | firstBuffer | | secondBufferLength | ignored | | | secondBuffer | Only exists if MetadataPacket and will copy over "cause" buffer | secondBuffer | h6. DevicePacket to SIAM Byte Array This is done by the SIAM Exportable Packet class || DevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataRef | direct copy | metadataRef | | parentId | direct copy | parentId | | recordType | direct copy | recordType | | | This is set based on what type of packet: # If MetadataPacket, set to 0x0101 # If SensorDataPacket, set to 0x0102 # If DeviceMessagePacket, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | firstBufferLength | direct copy (see note for first buffer) | firstBufferLength | | firstBuffer | This depends on the type of packet: # If MetadataPacket, "cause" bytes are copied over # If SensorDataPacket, "dataMessage" bytes are copied over # If DeviceMessagePacket, "message" bytes are copied over | firstBuffer | | secondBufferLength | only valid with MetadataPacket, but is copied directly over | secondBufferLength | | secondBuffer | only valid with MetadataPacket, and the "buffer" bytes are copied over | secondBuffer | h6. SSDSDevicePacket to SIAM Byte Array *Originally done in TransmogrifyMDB by calling SSDSDevicePacket.convertToPublishableVersion3ByteArray before passing byte array to method to translate to SSDS format and then send to ingest* || SSDSDevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataSequenceNumber | direct copy | metadataRef | | parentID | direct copy | parentID | | recordType | # If packetType = 0, set recordType = 0 # If packetType = 1, set recordType = recordType # If packetType = 2, set recordType = recordType | recordType | | | This is set based on what type of packet: # If packetType = 0, set to 0x0101 # If packetType = 1, set to 0x0102 # If packetType = 2, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | | This depends on the type of packet: # If packetType = 0, set to length of "otherBuffer" # If packetType = 1, set to length of "dataBuffer" # If packetType = 2, set to length of "dataBuffer" | firstBufferLength | | other/dataBuffer | This depends on the type of packet: # If packetType = 0, set to bytes from "otherBuffer" # If packetType = 1, set to bytes from "dataBuffer" # If packetType = 2, set to bytes from "dataBuffer" | firstBuffer | | | This only exists if it is packetType = 0 then it is set to the length of the "dataBuffer" | secondBufferLength | | dataBuffer | This only exists if it is packetType = 0 then it is set to the byte from the "dataBuffer" | secondBuffer | h5. SSDSDevicePacket to SSDS Byte Array *Originally done in SSDSDevicePacket.convertToVersion3ByteArray* || SSDSDevicePacket || Translation Rule || SSDS Byte Array || | sourceID | direct copy | sourceID | | systemTime | ignored during the translation directly, but used through getter methods for seconds and nanoseconds | | | timestampSeconds | direct copy (note that this is _sort of_ a direct copy, there are getter methods on SSDSDevicePacket that convert the systemTime to seconds and nanoseconds when called). | timestampSeconds | | timestampNanoseconds | direct copy (note that this is _sort of_ a direct copy, there are getter methods on SSDSDevicePacket that convert the systemTime to seconds and nanoseconds when called). | timestampNanoseconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | ignored | | | parentID | ignored | | | recordType | If packetType = 0, set packetSubType to 0, otherwise set to recordType | packetSubType | | packetType | Depends on packetType: # If packetType = 0, set to 1 # If packetType = 1, set to 0 # If packetType = 2, set to 4 | packetType | | metadataSequenceNumber | direct copy | metadataSequenceNumber | | dataDescriptionVersion | direct copy | dataDescriptionVersion | | platformID | direct copy | parentID | | | copy length of dataBuffer | firstBufferLength | | dataBuffer | direct copy | firstBuffer | | | copy length of otherBuffer | secondBufferLength | | otherBuffer | direct copy | secondBuffer | h6. SIAM Byte Array to SSDS Byte Array *Originally done in TransmogrifyMDB in checkAndPublishBytes method* || SIAM Byte Array || Translation Rules || SSDS Byte Array || | EX_DEVICEPACKET | ignored | | | serialVersionUID | ignored | | | sourceID | direct copy | sourceID | | | Depending on EX_XXXXXXXPACKET: # If MetadataPacket, set packetType to 1 # If SensorDataPacket, set packetType to 0 # If DeviceMessagePacket, set packetType to 4 | packetType | | | This was set using the SIAMMetadataTracker that tried to keep track of real version numbers based on XML in payload | metadataSequencNumber | | systemTime | Split into timestampSeconds and timestampNanoseconds | # timestampSeconds # timestampNanoSeconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | direct copy | dataDescriptionVersion | | parentId | direct copy | parentID | | recordType | If MetadataPacket (determined from EX_XXXXXXXPACKET), recordType set to 0, otherwise set to recordType | packetSubType | | EX_XXXXXXXPACKET | ignored in storage, but used in logic | | | serialVersionUID | ignored | | | first/secondBufferLength | If MetadataPacket (determined from EX_XXXXXXXPACKET), firstBufferLength is set to secondBufferLength so we can flip the "cause" and "buffer" bytes because it just made more sense since the cause was rarely populated. Otherwise set to firstBufferLength | firstBufferLength | | first/secondBuffer | If MetadataPacket (determined from EX_XXXXXXXPACKET), firstBuffer is set to secondBuffer so we can flip the "cause" and "buffer" bytes because it just made more sense since the cause was rarely populated. Otherwise set to firstBuffer | firstBuffer | | firstBufferLength | If MetadataPacket (determined from EX_XXXXXXXPACKET), secondBufferLength is set to firstBufferLength to flip "cause" and "buffer" bytes | secondBufferLength | | firstBuffer | If MetadataPacket (determined from EX_XXXXXXXPACKET), secondBuffer is set to firstBuffer to flip "cause" and "buffer" bytes | secondBuffer | h5. Exploration of Upgrade of Ingest/Transmogrify to AMQP In an effort to allow non-Java clients to send data to SSDS in the form of messages and to upgrade the messaging system to a technology that is more scalable and higher performance, an investigation of AMQP implementations was done. # [Qpid Exploration] ]]> 16416860 8388627 8355860 10388621 Databases->SSDS_Data->Tables and you should see something like this: ]]> 10355895 17268798 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! (lots of other output skipped) SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the input and output time axes: {noformat} {noformat} ]]> 17236032 16449624 metadataRef # metadataRef->metadataSequenceNumber # metadataRef->dataDescriptionID | # metadataRef # metadataSequenceNumber # dataDescriptionID | | parentId | direct copy: # parentId->parentId # parentId->platformID | # parentId # platformID | | recordType | # MetadataPacket: recordType to 0 # Other: direct copy | recordType | | | # If MetadataPacket, packetType = 0 # If SensorDataPacket, packetType = 1 # If DeviceMessagePacket, packetType = 2 | packetType | | firstBufferLength | ignored | | | firstBuffer | First buffer depends on which type of packet # If MetadataPacket, copy "bytes" buffer # If SensorDataPacket, copy "dataBuffer" # If DeviceMessagePacket, copy "message" | firstBuffer | | secondBufferLength | ignored | | | secondBuffer | Only exists if MetadataPacket and will copy over "cause" buffer | secondBuffer | h6. DevicePacket to SIAM Byte Array This is done by the SIAM Exportable Packet class || DevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataRef | direct copy | metadataRef | | parentId | direct copy | parentId | | recordType | direct copy | recordType | | | This is set based on what type of packet: # If MetadataPacket, set to 0x0101 # If SensorDataPacket, set to 0x0102 # If DeviceMessagePacket, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | firstBufferLength | direct copy (see note for first buffer) | firstBufferLength | | firstBuffer | This depends on the type of packet: # If MetadataPacket, "cause" bytes are copied over # If SensorDataPacket, "dataMessage" bytes are copied over # If DeviceMessagePacket, "message" bytes are copied over | firstBuffer | | secondBufferLength | only valid with MetadataPacket, but is copied directly over | secondBufferLength | | secondBuffer | only valid with MetadataPacket, and the "buffer" bytes are copied over | secondBuffer | h6. SSDSDevicePacket to SIAM Byte Array *Originally done in TransmogrifyMDB by calling SSDSDevicePacket.convertToPublishableVersion3ByteArray before passing byte array to method to translate to SSDS format and then send to ingest* || SSDSDevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataSequenceNumber | direct copy | metadataRef | | parentID | direct copy | parentID | | recordType | # If packetType = 0, set recordType = 0 # If packetType = 1, set recordType = recordType # If packetType = 2, set recordType = recordType | recordType | | | This is set based on what type of packet: # If packetType = 0, set to 0x0101 # If packetType = 1, set to 0x0102 # If packetType = 2, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | | This depends on the type of packet: # If packetType = 0, set to length of "otherBuffer" # If packetType = 1, set to length of "dataBuffer" # If packetType = 2, set to length of "dataBuffer" | firstBufferLength | | other/dataBuffer | This depends on the type of packet: # If packetType = 0, set to bytes from "otherBuffer" # If packetType = 1, set to bytes from "dataBuffer" # If packetType = 2, set to bytes from "dataBuffer" | firstBuffer | | | This only exists if it is packetType = 0 then it is set to the length of the "dataBuffer" | secondBufferLength | | dataBuffer | This only exists if it is packetType = 0 then it is set to the byte from the "dataBuffer" | secondBuffer | h5. SSDSDevicePacket to SSDS Byte Array *Originally done in SSDSDevicePacket.convertToVersion3ByteArray* || SSDSDevicePacket || Translation Rule || SSDS Byte Array || | sourceID | direct copy | sourceID | | systemTime | ignored during the translation directly, but used through getter methods for seconds and nanoseconds | | | timestampSeconds | direct copy (note that this is _sort of_ a direct copy, there are getter methods on SSDSDevicePacket that convert the systemTime to seconds and nanoseconds when called). | timestampSeconds | | timestampNanoseconds | direct copy (note that this is _sort of_ a direct copy, there are getter methods on SSDSDevicePacket that convert the systemTime to seconds and nanoseconds when called). | timestampNanoseconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | ignored | | | parentID | ignored | | | recordType | If packetType = 0, set packetSubType to 0, otherwise set to recordType | packetSubType | | packetType | Depends on packetType: # If packetType = 0, set to 1 # If packetType = 1, set to 0 # If packetType = 2, set to 4 | packetType | | metadataSequenceNumber | direct copy | metadataSequenceNumber | | dataDescriptionVersion | direct copy | dataDescriptionVersion | | platformID | direct copy | parentID | | | copy length of dataBuffer | firstBufferLength | | dataBuffer | direct copy | firstBuffer | | | copy length of otherBuffer | secondBufferLength | | otherBuffer | direct copy | secondBuffer | h6. SIAM Byte Array to SSDS Byte Array *Originally done in TransmogrifyMDB in checkAndPublishBytes method* || SIAM Byte Array || Translation Rules || SSDS Byte Array || | EX_DEVICEPACKET | ignored | | | serialVersionUID | ignored | | | sourceID | direct copy | sourceID | | | Depending on EX_XXXXXXXPACKET: # If MetadataPacket, set packetType to 1 # If SensorDataPacket, set packetType to 0 # If DeviceMessagePacket, set packetType to 4 | packetType | | | This was set using the SIAMMetadataTracker that tried to keep track of real version numbers based on XML in payload | metadataSequencNumber | | systemTime | Split into timestampSeconds and timestampNanoseconds | # timestampSeconds # timestampNanoSeconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | direct copy | dataDescriptionVersion | | parentId | direct copy | parentID | | recordType | If MetadataPacket (determined from EX_XXXXXXXPACKET), recordType set to 0, otherwise set to recordType | packetSubType | | EX_XXXXXXXPACKET | ignored in storage, but used in logic | | | serialVersionUID | ignored | | | first/secondBufferLength | If MetadataPacket (determined from EX_XXXXXXXPACKET), firstBufferLength is set to secondBufferLength so we can flip the "cause" and "buffer" bytes because it just made more sense since the cause was rarely populated. Otherwise set to firstBufferLength | firstBufferLength | | first/secondBuffer | If MetadataPacket (determined from EX_XXXXXXXPACKET), firstBuffer is set to secondBuffer so we can flip the "cause" and "buffer" bytes because it just made more sense since the cause was rarely populated. Otherwise set to firstBuffer | firstBuffer | | firstBufferLength | If MetadataPacket (determined from EX_XXXXXXXPACKET), secondBufferLength is set to firstBufferLength to flip "cause" and "buffer" bytes | secondBufferLength | | firstBuffer | If MetadataPacket (determined from EX_XXXXXXXPACKET), secondBuffer is set to firstBuffer to flip "cause" and "buffer" bytes | secondBuffer | h5. Exploration of Upgrade of Ingest/Transmogrify to AMQP In an effort to allow non-Java clients to send data to SSDS in the form of messages and to upgrade the messaging system to a technology that is more scalable and higher performance, an investigation of AMQP implementations was done. # [Qpid Exploration] ]]> 16416861 8388628 8355861 17268792 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row.  Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for dagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log # Examine the data and plots for the individual microcats on the current_qcPlots.html web page ]]> 17236026 10388617 10355891 17268794 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log # Examine the data and plots for the individual microcats on the current_qcPlots.html web page ]]> 17236028 17268788 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row.  Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. ]]> 17236022 17268790 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row.  Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for dagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log ]]> 17236024 17268783 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data from file ]]> 17236017 3178563 3113031 17268786 17236020 17268785 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the [TS: ] row.]]> 17236019 17268779 17236013 10388638 10355912 17268781 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="28-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 72 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 28-MAY-2011 04:30 / 5097: 11.55 .... .... .... .... .... .... .... .... .... .... 28-MAY-2011 05:30 / 5098: 11.50 .... .... .... .... .... .... .... .... .... .... 28-MAY-2011 06:30 / 5099: 11.48 .... .... .... .... .... .... .... .... .... .... 28-MAY-2011 07:30 / 5100: 11.44 .... .... .... .... .... .... .... .... .... .... 28-MAY-2011 08:30 / 5101: 11.41 .... .... .... .... .... .... .... .... .... .... 28-MAY-2011 09:30 / 5102: 11.36 .... .... .... .... .... .... .... .... .... .... 28-MAY-2011 10:30 / 5103: 11.36 .... .... .... .... .... .... .... .... .... .... 28-MAY-2011 11:30 / 5104: 11.52 .... .... .... .... .... .... .... .... .... .... 28-MAY-2011 12:30 / 5105: 11.69 .... .... .... .... .... .... .... .... .... .... 28-MAY-2011 13:30 / 5106: 11.82 .... .... .... .... .... .... .... .... .... .... 28-MAY-2011 14:30 / 5107: 11.80 .... .... .... .... .... .... .... .... .... .... 28-MAY-2011 15:30 / 5108: 11.85 .... .... 10.14 9.31 8.95 8.77 8.28 8.03 7.69 7.55 28-MAY-2011 16:30 / 5109: 12.01 .... .... 9.78 9.42 8.94 8.77 8.29 8.08 7.65 7.55 28-MAY-2011 17:30 / 5110: 12.04 .... .... 9.71 9.25 8.94 8.80 8.33 8.09 7.69 7.53 28-MAY-2011 18:30 / 5111: 12.24 .... .... 9.74 9.27 8.92 8.81 8.37 8.09 7.73 7.47 28-MAY-2011 19:30 / 5112: 12.37 .... .... 9.83 9.31 8.92 8.81 8.26 8.08 7.71 7.49 28-MAY-2011 20:30 / 5113: 12.44 .... .... .... .... .... .... .... .... .... .... 28-MAY-2011 21:30 / 5114: 12.57 .... .... .... .... .... .... .... .... .... .... 28-MAY-2011 22:30 / 5115: 12.19 .... .... .... .... .... .... .... .... .... .... 28-MAY-2011 23:30 / 5116: 12.09 .... .... .... .... .... .... .... .... .... .... 29-MAY-2011 00:30 / 5117: 12.07 .... .... .... .... .... .... .... .... .... .... 29-MAY-2011 01:30 / 5118: 12.12 .... .... .... .... .... .... .... .... .... .... 29-MAY-2011 02:30 / 5119: 12.11 .... .... .... .... .... .... .... .... .... .... 29-MAY-2011 03:30 / 5120: 12.15 .... .... .... .... .... .... .... .... .... .... 29-MAY-2011 04:30 / 5121: 12.11 .... .... .... .... .... .... .... .... .... .... 29-MAY-2011 05:30 / 5122: 11.82 .... .... .... .... .... .... .... .... .... .... 29-MAY-2011 06:30 / 5123: 11.75 .... .... .... .... .... .... .... .... .... .... 29-MAY-2011 07:30 / 5124: 11.70 .... .... .... .... .... .... .... .... .... .... 29-MAY-2011 08:30 / 5125: 11.63 .... .... .... .... .... .... .... .... .... .... 29-MAY-2011 09:30 / 5126: 11.58 .... .... .... .... .... .... .... .... .... .... 29-MAY-2011 10:30 / 5127: 11.53 .... .... .... .... .... .... .... .... .... .... 29-MAY-2011 11:30 / 5128: 11.47 .... .... .... .... .... .... .... .... .... .... 29-MAY-2011 12:30 / 5129: 11.44 .... .... 9.54 9.15 8.97 8.71 8.27 8.02 7.80 7.54 29-MAY-2011 13:30 / 5130: 11.40 .... .... 9.49 9.17 8.81 8.50 8.23 8.01 7.78 7.51 29-MAY-2011 14:30 / 5131: 11.37 .... .... 9.43 9.15 8.97 8.55 8.14 7.93 7.70 7.50 29-MAY-2011 15:30 / 5132: 11.36 .... .... 9.35 9.19 8.97 8.64 8.20 7.91 7.66 7.50 29-MAY-2011 16:30 / 5133: 11.35 .... .... 9.52 9.21 8.97 8.73 8.23 7.91 7.66 7.53 29-MAY-2011 17:30 / 5134: 11.37 .... .... 9.56 9.16 9.04 8.79 8.22 7.91 7.70 7.50 29-MAY-2011 18:30 / 5135: 11.38 .... .... 9.58 9.28 9.07 8.77 8.23 7.98 7.71 7.52 29-MAY-2011 19:30 / 5136: 11.36 .... .... 9.65 9.27 9.01 8.73 8.25 8.02 7.67 7.47 29-MAY-2011 20:30 / 5137: 11.36 .... .... 9.60 9.25 9.00 8.70 8.26 8.03 7.75 7.54 29-MAY-2011 21:30 / 5138: 11.35 .... .... 9.58 9.27 8.95 8.70 8.25 8.02 7.80 7.56 29-MAY-2011 22:30 / 5139: 11.37 .... .... 9.59 9.01 8.81 8.65 8.19 8.00 7.71 7.56 29-MAY-2011 23:30 / 5140: 11.38 .... .... 9.52 9.11 8.87 8.64 8.19 8.02 7.76 7.53 30-MAY-2011 00:30 / 5141: 11.34 .... .... 9.53 9.28 8.97 8.64 8.18 8.03 7.79 7.51 30-MAY-2011 01:30 / 5142: 11.20 .... .... 9.50 9.20 8.94 8.63 8.16 8.04 7.81 7.55 30-MAY-2011 02:30 / 5143: 10.99 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 03:30 / 5144: 10.89 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....  {noformat}]]> 17236015 10388637 10355911 8388626 8355859 17268775 17236009 8388624 8355857 17268777 17236011 10388628 Databases->SSDS_Data->Tables and you should see something like this: !Figure 1.jpg|align=centre! You are looking for the *DeviceQCPlotConfig* table and if you scroll down and right-click on it, then choose 'Open Table->Return all Rows' you will see the listing of all the configurations for creating plots (Figure 2). !Figure 2.jpg|align=centre! Here is a list of the columns and what they mean: * DeviceID_FK * PacketSubType * RecordVariable_Name * NumberOfHoursBack * Chart_Type * X_Size * Y_Size * X_Axis_Label * Y_Axis_Label * Title * Page_Title * Page_URL_String * Image_URL_String * Data_Page_URL_String * Y_Max * Y_Min * Y_Axis_One_Or_Two * Use_Long_Variable_Name * Gps_Show_Anchor * Gps_Anchor_Latitude * Gps_Show_Watch_Circle * Gps_Watch_Circle_Diameter_Km * Gps_Scale_Chart_To_Fit_Data * GraphOn ]]> 10355902 17268773 17236007 8388659 8355892 8388661 8355896 5865697 5832939 5865695 5832937 5865694 5832936 8388652 8355885 17268864 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the change log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is [http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log] # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: [http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED] To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:`PSAL,return=lend`/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC !-> LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:5182/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the first 3 temperature values from the 1m and 10m input CTD data and the first 3 times of the output axis: {noformat} yes? list/l=1:3/d=1 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1338 at original sampling intervals FILENAME : m1_ctd0001_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 1 122W 1 27-OCT-2010 21:08:43 / 1: 14.01 27-OCT-2010 21:22:27 / 2: 14.10 27-OCT-2010 21:28:23 / 3: 14.10 yes? list/l=1:3/d=2 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 27-OCT-2010 20:11:17 / 1: 13.38 27-OCT-2010 20:21:16 / 2: 13.59 27-OCT-2010 20:31:16 / 3: 13.45 yes? show/l=1:3 axis time name axis # pts start end TIME TIME 5181 r 27-OCT-2010 20:30 31-MAY-2011 16:30 T0 = 01-JAN-1950 00:00:00 Axis span (to cell edges) = 215.875 L T TBOX TBOXLO TSTEP (DAYS) 1> 27-OCT-2010 20:30:00 0.0416667 27-OCT-2010 20:00:00 22214.85 2> 27-OCT-2010 21:30:00 0.0416667 27-OCT-2010 21:00:00 22214.9 3> 27-OCT-2010 22:30:00 0.0416667 27-OCT-2010 22:00:00 22214.94 {noformat} The problem seems to be that the 10m data begin at 20:11:17 before the 1m data at 21:08:43. The first write of the 1m data to the output netCDF file gives the file its shape with the KLIMITS= and LLIMITS= options. Writing the 10m data, which begins in before the bounds that were defined violates Ferret's axis ordering logic. To test this hypothesis edit the SAVE statement in the temporary .jnl file to make the start time for the 10m data 21:00, execute it and then examine the output file: {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 21:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc (Another ferret session) > ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 31-May-11 12:19 yes? use "/tmp/OS_MBARI-M1_20101027_R_TS.nc" yes? list/l=1:10 temp VARIABLE : Hourly sea_water_temperature (celsius) FILENAME : OS_MBARI-M1_20101027_R_TS.nc FILEPATH : /tmp/ SUBSET : 11 by 10 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 27-OCT-2010 21:30 / 1: 14.09 13.60 .... .... .... .... .... .... .... .... .... 27-OCT-2010 22:30 / 2: 14.17 13.62 .... .... .... .... .... .... .... .... .... 27-OCT-2010 23:30 / 3: 14.34 13.63 .... .... .... .... .... .... .... .... .... 28-OCT-2010 00:30 / 4: 14.28 13.69 .... .... .... .... .... .... .... .... .... 28-OCT-2010 01:30 / 5: 14.22 13.72 .... .... .... .... .... .... .... .... .... 28-OCT-2010 02:30 / 6: 14.21 13.78 .... .... .... .... .... .... .... .... .... 28-OCT-2010 03:30 / 7: 14.21 13.76 .... .... .... .... .... .... .... .... .... 28-OCT-2010 04:30 / 8: 14.09 13.79 .... .... .... .... .... .... .... .... .... 28-OCT-2010 05:30 / 9: 14.01 13.77 .... .... .... .... .... .... .... .... .... 28-OCT-2010 06:30 / 10: 13.89 13.76 .... .... .... .... .... .... .... .... .... yes? {noformat} O.K.\! That looks good. The 10m data got written. Now to fix it for production. This deployment was a special case where the 1m data start after the 10m data. Typically, it's most convenient to have all of the microcats have the same deployment start time. This makes the follow on data processing much simpler. All of the deployment start times in SSDS_METADTA for the M1 201010 deployment were adjusted to be the same as the start time for the Mooring deployment. Performing this step is now part of the [standard operating procedure for perming mooring turns|SSDS:OASIS Mooring turn]. Once SSDS_METADATA has been updated the individual instrument netCDF files must be created by DStoNetCDF.pl before combineTS.pl in run. There are still some problems with gappy data at 40m and below in the \_TS file. To debug these we'll make another copy of the OS_MBARI-M1_20101027_R_TS.jnl to /tmp and edit it to go through the processing of the 40m data saving the data to a netCDF file in /tmp so as not to disturb the production processing. We also comment out all of the "CANCEL DATA 1" statements so that we can examine data from the files. This file is executed and we see no errors. We need to examine the input data and the Ferret variables that are constructed for the gridding. Here are the commands from the .jnl that perform the the gridding for the 40m data: {noformat} ! ! Remove temperature outliers before gridding ! LET Temperature_QFLAG = IF Temperature GT 2 AND Temperature LT 20 THEN 1 ELSE (-99999) SET VAR/BAD=-99999 Temperature_QFLAG LET Temperature_QC = Temperature_QFLAG * Temperature ! ! Compute my own mean of the data, making sure to assign missing values in the gaps in the Gap FLAG ! Allow at least 1 data point in each destination cell - there are usually 6 for 10 minute data ! LET Temperature_GOOD = Temperature_QC[gt=TIME@SUM] / Temperature_QC[gt=TIME@NGD] LET Temperature_GFLAG = IF Temperature[gt=TIME@NGD,gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] LT 1 THEN (-99999) ELSE 1 SET VAR/BAD=-99999 Temperature_GFLAG LET TEMP = Temperature_GFLAG * Temperature_GOOD[gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] {noformat} Here are some sample Ferret commands for plotting these data from the session that just executed the OS_MBARI-M1_20101027_R_TS.jnl script: {noformat} yes? go "OS_MBARI-M1_20101027_R_TS.jnl" yes? set region/t=1-may-2011:1-jun-2011 yes? plot Temperature_QC yes? plot/ov/symbol=1 temperature yes? set win 2 yes? plot Temperature_QC[gt=TIME@NGD] yes? plot TEMPERATURE_QC[GT=TIME@SUM] {noformat} These last two plot commands indicate the source of the problem of the gappy data from the inductive modem microcats. Another big clue is the comment in the .jnl file referring to the 10-minute input data and having at least 6 good values within each gridding cell. As can be seen in the plot we often have less than 1 for the @NGD value for the 40m data. These data are hourly and we should be using a different gridding algorithm for them. !m1_ngd_sum.gif|thumbnail! The October 2010 M1 deployment was the first one where all the inductive modem connected CTDs were logged as separate instruments, and the first deployment where the serial microcat processing software was used to process the inductive modem microcat data. The fix for this problem appears to be to do a different gridding for the hourly data from 40m and below. This change was implemented in the code on 2 June 2011 using the Ferret same gridding transform (@MAX) that was previously used for the inductive modem data processing.  Here is the after figure: !wind_SEA_WATER_TEMPERATURE_HR_last30_after.gif|thumbnail! The subsurface data looks a lot better, but there are still some gaps.  If the gaps extend across the whole water column encompassing both the inductive modem and serial microcats then perhaps the gap is due to an interruption in data processing through oasisToSSDS.  This can be remedied by reprocessing the telemetered data by following the README instructions in /u/ssdsadmin/dev/DPforSSDS/oasis/scripts on elvis.   It is worthwhile to do another check on the quality of the gridding that is now being performed.  The following Ferret commands will show the input data compared to the gridded data:    ]]> 17236100 17268866 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the change log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is [http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log] # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: [http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED] To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:`PSAL,return=lend`/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC !-> LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:5182/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the first 3 temperature values from the 1m and 10m input CTD data and the first 3 times of the output axis: {noformat} yes? list/l=1:3/d=1 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1338 at original sampling intervals FILENAME : m1_ctd0001_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 1 122W 1 27-OCT-2010 21:08:43 / 1: 14.01 27-OCT-2010 21:22:27 / 2: 14.10 27-OCT-2010 21:28:23 / 3: 14.10 yes? list/l=1:3/d=2 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 27-OCT-2010 20:11:17 / 1: 13.38 27-OCT-2010 20:21:16 / 2: 13.59 27-OCT-2010 20:31:16 / 3: 13.45 yes? show/l=1:3 axis time name axis # pts start end TIME TIME 5181 r 27-OCT-2010 20:30 31-MAY-2011 16:30 T0 = 01-JAN-1950 00:00:00 Axis span (to cell edges) = 215.875 L T TBOX TBOXLO TSTEP (DAYS) 1> 27-OCT-2010 20:30:00 0.0416667 27-OCT-2010 20:00:00 22214.85 2> 27-OCT-2010 21:30:00 0.0416667 27-OCT-2010 21:00:00 22214.9 3> 27-OCT-2010 22:30:00 0.0416667 27-OCT-2010 22:00:00 22214.94 {noformat} The problem seems to be that the 10m data begin at 20:11:17 before the 1m data at 21:08:43. The first write of the 1m data to the output netCDF file gives the file its shape with the KLIMITS= and LLIMITS= options. Writing the 10m data, which begins in before the bounds that were defined violates Ferret's axis ordering logic. To test this hypothesis edit the SAVE statement in the temporary .jnl file to make the start time for the 10m data 21:00, execute it and then examine the output file: {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 21:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc (Another ferret session) > ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 31-May-11 12:19 yes? use "/tmp/OS_MBARI-M1_20101027_R_TS.nc" yes? list/l=1:10 temp VARIABLE : Hourly sea_water_temperature (celsius) FILENAME : OS_MBARI-M1_20101027_R_TS.nc FILEPATH : /tmp/ SUBSET : 11 by 10 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 27-OCT-2010 21:30 / 1: 14.09 13.60 .... .... .... .... .... .... .... .... .... 27-OCT-2010 22:30 / 2: 14.17 13.62 .... .... .... .... .... .... .... .... .... 27-OCT-2010 23:30 / 3: 14.34 13.63 .... .... .... .... .... .... .... .... .... 28-OCT-2010 00:30 / 4: 14.28 13.69 .... .... .... .... .... .... .... .... .... 28-OCT-2010 01:30 / 5: 14.22 13.72 .... .... .... .... .... .... .... .... .... 28-OCT-2010 02:30 / 6: 14.21 13.78 .... .... .... .... .... .... .... .... .... 28-OCT-2010 03:30 / 7: 14.21 13.76 .... .... .... .... .... .... .... .... .... 28-OCT-2010 04:30 / 8: 14.09 13.79 .... .... .... .... .... .... .... .... .... 28-OCT-2010 05:30 / 9: 14.01 13.77 .... .... .... .... .... .... .... .... .... 28-OCT-2010 06:30 / 10: 13.89 13.76 .... .... .... .... .... .... .... .... .... yes? {noformat} O.K.\! That looks good. The 10m data got written. Now to fix it for production. This deployment was a special case where the 1m data start after the 10m data. Typically, it's most convenient to have all of the microcats have the same deployment start time. This makes the follow on data processing much simpler. All of the deployment start times in SSDS_METADTA for the M1 201010 deployment were adjusted to be the same as the start time for the Mooring deployment. Performing this step is now part of the [standard operating procedure for perming mooring turns|SSDS:OASIS Mooring turn]. Once SSDS_METADATA has been updated the individual instrument netCDF files must be created by DStoNetCDF.pl before combineTS.pl in run. There are still some problems with gappy data at 40m and below in the \_TS file. To debug these we'll make another copy of the OS_MBARI-M1_20101027_R_TS.jnl to /tmp and edit it to go through the processing of the 40m data saving the data to a netCDF file in /tmp so as not to disturb the production processing. We also comment out all of the "CANCEL DATA 1" statements so that we can examine data from the files. This file is executed and we see no errors. We need to examine the input data and the Ferret variables that are constructed for the gridding. Here are the commands from the .jnl that perform the the gridding for the 40m data: {noformat} ! ! Remove temperature outliers before gridding ! LET Temperature_QFLAG = IF Temperature GT 2 AND Temperature LT 20 THEN 1 ELSE (-99999) SET VAR/BAD=-99999 Temperature_QFLAG LET Temperature_QC = Temperature_QFLAG * Temperature ! ! Compute my own mean of the data, making sure to assign missing values in the gaps in the Gap FLAG ! Allow at least 1 data point in each destination cell - there are usually 6 for 10 minute data ! LET Temperature_GOOD = Temperature_QC[gt=TIME@SUM] / Temperature_QC[gt=TIME@NGD] LET Temperature_GFLAG = IF Temperature[gt=TIME@NGD,gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] LT 1 THEN (-99999) ELSE 1 SET VAR/BAD=-99999 Temperature_GFLAG LET TEMP = Temperature_GFLAG * Temperature_GOOD[gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] {noformat} Here are some sample Ferret commands for plotting these data from the session that just executed the OS_MBARI-M1_20101027_R_TS.jnl script: {noformat} yes? go "OS_MBARI-M1_20101027_R_TS.jnl" yes? set region/t=1-may-2011:1-jun-2011 yes? plot Temperature_QC yes? plot/ov/symbol=1 temperature yes? set win 2 yes? plot Temperature_QC[gt=TIME@NGD] yes? plot TEMPERATURE_QC[GT=TIME@SUM] {noformat} These last two plot commands indicate the source of the problem of the gappy data from the inductive modem microcats. Another big clue is the comment in the .jnl file referring to the 10-minute input data and having at least 6 good values within each gridding cell. As can be seen in the plot we often have less than 1 for the @NGD value for the 40m data. These data are hourly and we should be using a different gridding algorithm for them. !m1_ngd_sum.gif|thumbnail! The October 2010 M1 deployment was the first one where all the inductive modem connected CTDs were logged as separate instruments, and the first deployment where the serial microcat processing software was used to process the inductive modem microcat data. The fix for this problem appears to be to do a different gridding for the hourly data from 40m and below. This change was implemented in the code on 2 June 2011 using the Ferret same gridding transform (@MAX) that was previously used for the inductive modem data processing.  Here is the after figure: !wind_SEA_WATER_TEMPERATURE_HR_last30_after.gif|thumbnail! The subsurface data looks a lot better, but there are still some gaps.  If the gaps extend across the whole water column encompassing both the inductive modem and serial microcats then perhaps the gap is due to an interruption in data processing through oasisToSSDS.  This can be remedied by reprocessing the telemetered data by following the README instructions in /u/ssdsadmin/dev/DPforSSDS/oasis/scripts on elvis.   It is worthwhile to do another check on the quality of the gridding that is now being performed.  The following Ferret commands will show the input data compared to the gridded data: {noformat} USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" SET REGION/T="29-May-2011 18:13":"02-Jun-2011 18:13" shade SEA_WATER_TEMPERATURE_HR set win 2 plot/k=1/vlimits=7:13 SEA_WATER_TEMPERATURE_HR repeat/k=2:11 plot/ov SEA_WATER_TEMPERATURE_HR {noformat}]]> 17236102 17268852 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the change log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is [http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log] # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: [http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED] To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:`PSAL,return=lend`/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC !-> LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:5182/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the first 3 temperature values from the 1m and 10m input CTD data and the first 3 times of the output axis: {noformat} yes? list/l=1:3/d=1 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1338 at original sampling intervals FILENAME : m1_ctd0001_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 1 122W 1 27-OCT-2010 21:08:43 / 1: 14.01 27-OCT-2010 21:22:27 / 2: 14.10 27-OCT-2010 21:28:23 / 3: 14.10 yes? list/l=1:3/d=2 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 27-OCT-2010 20:11:17 / 1: 13.38 27-OCT-2010 20:21:16 / 2: 13.59 27-OCT-2010 20:31:16 / 3: 13.45 yes? show/l=1:3 axis time name axis # pts start end TIME TIME 5181 r 27-OCT-2010 20:30 31-MAY-2011 16:30 T0 = 01-JAN-1950 00:00:00 Axis span (to cell edges) = 215.875 L T TBOX TBOXLO TSTEP (DAYS) 1> 27-OCT-2010 20:30:00 0.0416667 27-OCT-2010 20:00:00 22214.85 2> 27-OCT-2010 21:30:00 0.0416667 27-OCT-2010 21:00:00 22214.9 3> 27-OCT-2010 22:30:00 0.0416667 27-OCT-2010 22:00:00 22214.94 {noformat} The problem seems to be that the 10m data begin at 20:11:17 before the 1m data at 21:08:43. The first write of the 1m data to the output netCDF file gives the file its shape with the KLIMITS= and LLIMITS= options. Writing the 10m data, which begins in before the bounds that were defined violates Ferret's axis ordering logic. To test this hypothesis edit the SAVE statement in the temporary .jnl file to make the start time for the 10m data 21:00, execute it and then examine the output file: {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 21:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc (Another ferret session) > ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 31-May-11 12:19 yes? use "/tmp/OS_MBARI-M1_20101027_R_TS.nc" yes? list/l=1:10 temp VARIABLE : Hourly sea_water_temperature (celsius) FILENAME : OS_MBARI-M1_20101027_R_TS.nc FILEPATH : /tmp/ SUBSET : 11 by 10 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 27-OCT-2010 21:30 / 1: 14.09 13.60 .... .... .... .... .... .... .... .... .... 27-OCT-2010 22:30 / 2: 14.17 13.62 .... .... .... .... .... .... .... .... .... 27-OCT-2010 23:30 / 3: 14.34 13.63 .... .... .... .... .... .... .... .... .... 28-OCT-2010 00:30 / 4: 14.28 13.69 .... .... .... .... .... .... .... .... .... 28-OCT-2010 01:30 / 5: 14.22 13.72 .... .... .... .... .... .... .... .... .... 28-OCT-2010 02:30 / 6: 14.21 13.78 .... .... .... .... .... .... .... .... .... 28-OCT-2010 03:30 / 7: 14.21 13.76 .... .... .... .... .... .... .... .... .... 28-OCT-2010 04:30 / 8: 14.09 13.79 .... .... .... .... .... .... .... .... .... 28-OCT-2010 05:30 / 9: 14.01 13.77 .... .... .... .... .... .... .... .... .... 28-OCT-2010 06:30 / 10: 13.89 13.76 .... .... .... .... .... .... .... .... .... yes? {noformat} O.K.\! That looks good. The 10m data got written. Now to fix it for production. This deployment was a special case where the 1m data start after the 10m data. Typically, it's most convenient to have all of the microcats have the same deployment start time. This makes the follow on data processing much simpler. All of the deployment start times in SSDS_METADTA for the M1 201010 deployment were adjusted to be the same as the start time for the Mooring deployment. Performing this step is now part of the standard operating procedure for perming mooring turns. Once SSDS_METADATA has been updated the individual instrument netCDF files must be created by DStoNetCDF.pl before combineTS.pl in run. There are still some problems with gappy data at 40m and below in the \_TS file. To debug these we'll make another copy of the OS_MBARI-M1_20101027_R_TS.jnl to /tmp and edit it to go through the processing of the 40m data saving the data to a netCDF file in /tmp so as not to disturb the production processing. We also comment out all of the "CANCEL DATA 1" statements so that we can examine data from the files. This file is executed and we see no errors. We need to examine the input data and the Ferret variables that are constructed for the gridding. Here are the commands from the .jnl that perform the the gridding for the 40m data: {noformat} ! ! Remove temperature outliers before gridding ! LET Temperature_QFLAG = IF Temperature GT 2 AND Temperature LT 20 THEN 1 ELSE (-99999) SET VAR/BAD=-99999 Temperature_QFLAG LET Temperature_QC = Temperature_QFLAG * Temperature ! ! Compute my own mean of the data, making sure to assign missing values in the gaps in the Gap FLAG ! Allow at least 1 data point in each destination cell - there are usually 6 for 10 minute data ! LET Temperature_GOOD = Temperature_QC[gt=TIME@SUM] / Temperature_QC[gt=TIME@NGD] LET Temperature_GFLAG = IF Temperature[gt=TIME@NGD,gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] LT 1 THEN (-99999) ELSE 1 SET VAR/BAD=-99999 Temperature_GFLAG LET TEMP = Temperature_GFLAG * Temperature_GOOD[gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] {noformat} Here are some sample Ferret commands for plotting these data from the session that just executed the OS_MBARI-M1_20101027_R_TS.jnl script: {noformat} yes? go "OS_MBARI-M1_20101027_R_TS.jnl" yes? set region/t=1-may-2011:1-jun-2011 yes? plot Temperature_QC yes? plot/ov/symbol=1 temperature yes? set win 2 yes? plot Temperature_QC[gt=TIME@NGD] yes? plot TEMPERATURE_QC[GT=TIME@SUM] {noformat} These last two plot commands indicate the source of the problem of the gappy data from the inductive modem microcats. Another big clue is the comment in the .jnl file referring to the 10-minute input data and having at least 6 good values within each gridding cell. As can be seen in the plot we often have less than 1 for the @NGD value for the 40m data. These data are hourly and we should be using a different gridding algorithm for them. !m1_ngd_sum.gif|thumbnail! The October 2010 M1 deployment was the first one where all the inductive modem connected CTDs were logged as separate instruments, and the first deployment where the serial microcat processing software was used to process the inductive modem microcat data. The fix for this problem appears to be to do a different gridding, say @AVE, for the hourly data from 40m and below.]]> 17236087 17268854 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the change log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is [http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log] # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: [http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED] To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:`PSAL,return=lend`/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC !-> LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:5182/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the first 3 temperature values from the 1m and 10m input CTD data and the first 3 times of the output axis: {noformat} yes? list/l=1:3/d=1 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1338 at original sampling intervals FILENAME : m1_ctd0001_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 1 122W 1 27-OCT-2010 21:08:43 / 1: 14.01 27-OCT-2010 21:22:27 / 2: 14.10 27-OCT-2010 21:28:23 / 3: 14.10 yes? list/l=1:3/d=2 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 27-OCT-2010 20:11:17 / 1: 13.38 27-OCT-2010 20:21:16 / 2: 13.59 27-OCT-2010 20:31:16 / 3: 13.45 yes? show/l=1:3 axis time name axis # pts start end TIME TIME 5181 r 27-OCT-2010 20:30 31-MAY-2011 16:30 T0 = 01-JAN-1950 00:00:00 Axis span (to cell edges) = 215.875 L T TBOX TBOXLO TSTEP (DAYS) 1> 27-OCT-2010 20:30:00 0.0416667 27-OCT-2010 20:00:00 22214.85 2> 27-OCT-2010 21:30:00 0.0416667 27-OCT-2010 21:00:00 22214.9 3> 27-OCT-2010 22:30:00 0.0416667 27-OCT-2010 22:00:00 22214.94 {noformat} The problem seems to be that the 10m data begin at 20:11:17 before the 1m data at 21:08:43. The first write of the 1m data to the output netCDF file gives the file its shape with the KLIMITS= and LLIMITS= options. Writing the 10m data, which begins in before the bounds that were defined violates Ferret's axis ordering logic. To test this hypothesis edit the SAVE statement in the temporary .jnl file to make the start time for the 10m data 21:00, execute it and then examine the output file: {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 21:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc (Another ferret session) > ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 31-May-11 12:19 yes? use "/tmp/OS_MBARI-M1_20101027_R_TS.nc" yes? list/l=1:10 temp VARIABLE : Hourly sea_water_temperature (celsius) FILENAME : OS_MBARI-M1_20101027_R_TS.nc FILEPATH : /tmp/ SUBSET : 11 by 10 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 27-OCT-2010 21:30 / 1: 14.09 13.60 .... .... .... .... .... .... .... .... .... 27-OCT-2010 22:30 / 2: 14.17 13.62 .... .... .... .... .... .... .... .... .... 27-OCT-2010 23:30 / 3: 14.34 13.63 .... .... .... .... .... .... .... .... .... 28-OCT-2010 00:30 / 4: 14.28 13.69 .... .... .... .... .... .... .... .... .... 28-OCT-2010 01:30 / 5: 14.22 13.72 .... .... .... .... .... .... .... .... .... 28-OCT-2010 02:30 / 6: 14.21 13.78 .... .... .... .... .... .... .... .... .... 28-OCT-2010 03:30 / 7: 14.21 13.76 .... .... .... .... .... .... .... .... .... 28-OCT-2010 04:30 / 8: 14.09 13.79 .... .... .... .... .... .... .... .... .... 28-OCT-2010 05:30 / 9: 14.01 13.77 .... .... .... .... .... .... .... .... .... 28-OCT-2010 06:30 / 10: 13.89 13.76 .... .... .... .... .... .... .... .... .... yes? {noformat} O.K.\! That looks good. The 10m data got written. Now to fix it for production. This deployment was a special case where the 1m data start after the 10m data. Typically, it's most convenient to have all of the microcats have the same deployment start time. This makes the follow on data processing much simpler. All of the deployment start times in SSDS_METADTA for the M1 201010 deployment were adjusted to be the same as the start time for the Mooring deployment. Performing this step is now part of the standard operating procedure for perming mooring turns. Once SSDS_METADATA has been updated the individual instrument netCDF files must be created by DStoNetCDF.pl before combineTS.pl in run. There are still some problems with gappy data at 40m and below in the \_TS file. To debug these we'll make another copy of the OS_MBARI-M1_20101027_R_TS.jnl to /tmp and edit it to go through the processing of the 40m data saving the data to a netCDF file in /tmp so as not to disturb the production processing. We also comment out all of the "CANCEL DATA 1" statements so that we can examine data from the files. This file is executed and we see no errors. We need to examine the input data and the Ferret variables that are constructed for the gridding. Here are the commands from the .jnl that perform the the gridding for the 40m data: {noformat} ! ! Remove temperature outliers before gridding ! LET Temperature_QFLAG = IF Temperature GT 2 AND Temperature LT 20 THEN 1 ELSE (-99999) SET VAR/BAD=-99999 Temperature_QFLAG LET Temperature_QC = Temperature_QFLAG * Temperature ! ! Compute my own mean of the data, making sure to assign missing values in the gaps in the Gap FLAG ! Allow at least 1 data point in each destination cell - there are usually 6 for 10 minute data ! LET Temperature_GOOD = Temperature_QC[gt=TIME@SUM] / Temperature_QC[gt=TIME@NGD] LET Temperature_GFLAG = IF Temperature[gt=TIME@NGD,gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] LT 1 THEN (-99999) ELSE 1 SET VAR/BAD=-99999 Temperature_GFLAG LET TEMP = Temperature_GFLAG * Temperature_GOOD[gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] {noformat} Here are some sample Ferret commands for plotting these data from the session that just executed the OS_MBARI-M1_20101027_R_TS.jnl script: {noformat} yes? go "OS_MBARI-M1_20101027_R_TS.jnl" yes? set region/t=1-may-2011:1-jun-2011 yes? plot Temperature_QC yes? plot/ov/symbol=1 temperature yes? set win 2 yes? plot Temperature_QC[gt=TIME@NGD] yes? plot TEMPERATURE_QC[GT=TIME@SUM] {noformat} These last two plot commands indicate the source of the problem of the gappy data from the inductive modem microcats. Another big clue is the comment in the .jnl file referring to the 10-minute input data and having at least 6 good values within each gridding cell. As can be seen in the plot we often have less than 1 for the @NGD value for the 40m data. These data are hourly and we should be using a different gridding algorithm for them. !m1_ngd_sum.gif|thumbnail! The October 2010 M1 deployment was the first one where all the inductive modem connected CTDs were logged as separate instruments, and the first deployment where the serial microcat processing software was used to process the inductive modem microcat data. The fix for this problem appears to be to do a different gridding for the hourly data from 40m and below. This change was implemented in the code on 2 June 2011 using the Ferret same gridding transform (@MAX) that was previously used for the inductive modem data processing.  Here is the after figure: !wind_SEA_WATER_TEMPERATURE_HR_last30_after.gif|thumbnail! The subsurface data looks a lot better, but there are still some gaps.  If the gaps extend across the whole water column encompassing both the inductive modem and serial microcats then perhaps the gap is due to an interruption in data processing through oasisToSSDS.  This can be remedied by reprocessing the telemetered data by following the README instructions in /u/ssdsadmin/dev/DPforSSDS/oasis/scripts on elvis.   It is worthwhile to do another check on the quality of the gridding that is now being performed.  The following Ferret commands will show the input data compared to the gridded data:    ]]> 17236089 17268844 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the change log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:`PSAL,return=lend`/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC !-> LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:5182/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the first 3 temperature values from the 1m and 10m input CTD data and the first 3 times of the output axis: {noformat} yes? list/l=1:3/d=1 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1338 at original sampling intervals FILENAME : m1_ctd0001_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 1 122W 1 27-OCT-2010 21:08:43 / 1: 14.01 27-OCT-2010 21:22:27 / 2: 14.10 27-OCT-2010 21:28:23 / 3: 14.10 yes? list/l=1:3/d=2 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 27-OCT-2010 20:11:17 / 1: 13.38 27-OCT-2010 20:21:16 / 2: 13.59 27-OCT-2010 20:31:16 / 3: 13.45 yes? show/l=1:3 axis time name axis # pts start end TIME TIME 5181 r 27-OCT-2010 20:30 31-MAY-2011 16:30 T0 = 01-JAN-1950 00:00:00 Axis span (to cell edges) = 215.875 L T TBOX TBOXLO TSTEP (DAYS) 1> 27-OCT-2010 20:30:00 0.0416667 27-OCT-2010 20:00:00 22214.85 2> 27-OCT-2010 21:30:00 0.0416667 27-OCT-2010 21:00:00 22214.9 3> 27-OCT-2010 22:30:00 0.0416667 27-OCT-2010 22:00:00 22214.94 {noformat} The problem seems to be that the 10m data begin at 20:11:17 before the 1m data at 21:08:43. The first write of the 1m data to the output netCDF file gives the file its shape with the KLIMITS= and LLIMITS= options. Writing the 10m data, which begins in before the bounds that were defined violates Ferret's axis ordering logic. To test this hypothesis edit the SAVE statement in the temporary .jnl file to make the start time for the 10m data 21:00, execute it and then examine the output file: {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 21:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc (Another ferret session) > ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 31-May-11 12:19 yes? use "/tmp/OS_MBARI-M1_20101027_R_TS.nc" yes? list/l=1:10 temp VARIABLE : Hourly sea_water_temperature (celsius) FILENAME : OS_MBARI-M1_20101027_R_TS.nc FILEPATH : /tmp/ SUBSET : 11 by 10 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 27-OCT-2010 21:30 / 1: 14.09 13.60 .... .... .... .... .... .... .... .... .... 27-OCT-2010 22:30 / 2: 14.17 13.62 .... .... .... .... .... .... .... .... .... 27-OCT-2010 23:30 / 3: 14.34 13.63 .... .... .... .... .... .... .... .... .... 28-OCT-2010 00:30 / 4: 14.28 13.69 .... .... .... .... .... .... .... .... .... 28-OCT-2010 01:30 / 5: 14.22 13.72 .... .... .... .... .... .... .... .... .... 28-OCT-2010 02:30 / 6: 14.21 13.78 .... .... .... .... .... .... .... .... .... 28-OCT-2010 03:30 / 7: 14.21 13.76 .... .... .... .... .... .... .... .... .... 28-OCT-2010 04:30 / 8: 14.09 13.79 .... .... .... .... .... .... .... .... .... 28-OCT-2010 05:30 / 9: 14.01 13.77 .... .... .... .... .... .... .... .... .... 28-OCT-2010 06:30 / 10: 13.89 13.76 .... .... .... .... .... .... .... .... .... yes? {noformat} O.K.! That looks good. The 10m data got written. Now to fix it for production. This deployment was a special case where the 1m data start after the 10m data. Typically, it's most convenient to have all of the microcats have the same deployment start time. This makes the follow on data processing much simpler. All of the deployment start times in SSDS_METADTA for the M1 201010 deployment were adjusted to be the same as the start time for the Mooring deployment. Performing this step is now part of the standard operating procedure for perming mooring turns. Once SSDS_METADATA has been updated the individual instrument netCDF files must be created by DStoNetCDF.pl before combineTS.pl in run. ]]> 17236079 17268846 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the change log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:`PSAL,return=lend`/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC !-> LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:5182/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the first 3 temperature values from the 1m and 10m input CTD data and the first 3 times of the output axis: {noformat} yes? list/l=1:3/d=1 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1338 at original sampling intervals FILENAME : m1_ctd0001_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 1 122W 1 27-OCT-2010 21:08:43 / 1: 14.01 27-OCT-2010 21:22:27 / 2: 14.10 27-OCT-2010 21:28:23 / 3: 14.10 yes? list/l=1:3/d=2 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 27-OCT-2010 20:11:17 / 1: 13.38 27-OCT-2010 20:21:16 / 2: 13.59 27-OCT-2010 20:31:16 / 3: 13.45 yes? show/l=1:3 axis time name axis # pts start end TIME TIME 5181 r 27-OCT-2010 20:30 31-MAY-2011 16:30 T0 = 01-JAN-1950 00:00:00 Axis span (to cell edges) = 215.875 L T TBOX TBOXLO TSTEP (DAYS) 1> 27-OCT-2010 20:30:00 0.0416667 27-OCT-2010 20:00:00 22214.85 2> 27-OCT-2010 21:30:00 0.0416667 27-OCT-2010 21:00:00 22214.9 3> 27-OCT-2010 22:30:00 0.0416667 27-OCT-2010 22:00:00 22214.94 {noformat} The problem seems to be that the 10m data begin at 20:11:17 before the 1m data at 21:08:43. The first write of the 1m data to the output netCDF file gives the file its shape with the KLIMITS= and LLIMITS= options. Writing the 10m data, which begins in before the bounds that were defined violates Ferret's axis ordering logic. To test this hypothesis edit the SAVE statement in the temporary .jnl file to make the start time for the 10m data 21:00, execute it and then examine the output file: {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 21:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc (Another ferret session) > ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 31-May-11 12:19 yes? use "/tmp/OS_MBARI-M1_20101027_R_TS.nc" yes? list/l=1:10 temp VARIABLE : Hourly sea_water_temperature (celsius) FILENAME : OS_MBARI-M1_20101027_R_TS.nc FILEPATH : /tmp/ SUBSET : 11 by 10 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 27-OCT-2010 21:30 / 1: 14.09 13.60 .... .... .... .... .... .... .... .... .... 27-OCT-2010 22:30 / 2: 14.17 13.62 .... .... .... .... .... .... .... .... .... 27-OCT-2010 23:30 / 3: 14.34 13.63 .... .... .... .... .... .... .... .... .... 28-OCT-2010 00:30 / 4: 14.28 13.69 .... .... .... .... .... .... .... .... .... 28-OCT-2010 01:30 / 5: 14.22 13.72 .... .... .... .... .... .... .... .... .... 28-OCT-2010 02:30 / 6: 14.21 13.78 .... .... .... .... .... .... .... .... .... 28-OCT-2010 03:30 / 7: 14.21 13.76 .... .... .... .... .... .... .... .... .... 28-OCT-2010 04:30 / 8: 14.09 13.79 .... .... .... .... .... .... .... .... .... 28-OCT-2010 05:30 / 9: 14.01 13.77 .... .... .... .... .... .... .... .... .... 28-OCT-2010 06:30 / 10: 13.89 13.76 .... .... .... .... .... .... .... .... .... yes? {noformat} O.K.! That looks good. The 10m data got written. Now to fix it for production. This deployment was a special case where the 1m data start after the 10m data. Typically, it's most convenient to have all of the microcats have the same deployment start time. This makes the follow on data processing much simpler. All of the deployment start times in SSDS_METADTA for the M1 201010 deployment were adjusted to be the same as the start time for the Mooring deployment. Performing this step is now part of the standard operating procedure for perming mooring turns. Once SSDS_METADATA has been updated the individual instrument netCDF files must be created by DStoNetCDF.pl before combineTS.pl in run. There are still some problems with gappy data at 40m and below in the _TS file. To debug these we'll make another copy of the OS_MBARI-M1_20101027_R_TS.jnl to /tmp and edit it to go through the processing of the 40m data saving the data to a netCDF file in /tmp so as not to disturb the production processing. We also comment out all of the "CANCEL DATA 1" statements so that we can examine data from the files. This file is executed and we see no errors. We need to examine the input data and the Ferret variables that are constructed for the gridding. Here are some sample Ferret commands for plotting these data from the session that just executed the OS_MBARI-M1_20101027_R_TS.jnl script: {noformat} yes? go "OS_MBARI-M1_20101027_R_TS.jnl yes? set region/t=1-may-2011:1-jun-2011 yes? plot Temperature_QC yes? plot/ov/symbol=1 temperature yes? set win 2 yes? plot Temperature_QC[gt=TIME@NGD] yes? plot TEMPERATURE_QC[GT=TIME@SUM] {noformat} ]]> 17236081 17268848 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the change log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:`PSAL,return=lend`/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC !-> LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:5182/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the first 3 temperature values from the 1m and 10m input CTD data and the first 3 times of the output axis: {noformat} yes? list/l=1:3/d=1 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1338 at original sampling intervals FILENAME : m1_ctd0001_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 1 122W 1 27-OCT-2010 21:08:43 / 1: 14.01 27-OCT-2010 21:22:27 / 2: 14.10 27-OCT-2010 21:28:23 / 3: 14.10 yes? list/l=1:3/d=2 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 27-OCT-2010 20:11:17 / 1: 13.38 27-OCT-2010 20:21:16 / 2: 13.59 27-OCT-2010 20:31:16 / 3: 13.45 yes? show/l=1:3 axis time name axis # pts start end TIME TIME 5181 r 27-OCT-2010 20:30 31-MAY-2011 16:30 T0 = 01-JAN-1950 00:00:00 Axis span (to cell edges) = 215.875 L T TBOX TBOXLO TSTEP (DAYS) 1> 27-OCT-2010 20:30:00 0.0416667 27-OCT-2010 20:00:00 22214.85 2> 27-OCT-2010 21:30:00 0.0416667 27-OCT-2010 21:00:00 22214.9 3> 27-OCT-2010 22:30:00 0.0416667 27-OCT-2010 22:00:00 22214.94 {noformat} The problem seems to be that the 10m data begin at 20:11:17 before the 1m data at 21:08:43. The first write of the 1m data to the output netCDF file gives the file its shape with the KLIMITS= and LLIMITS= options. Writing the 10m data, which begins in before the bounds that were defined violates Ferret's axis ordering logic. To test this hypothesis edit the SAVE statement in the temporary .jnl file to make the start time for the 10m data 21:00, execute it and then examine the output file: {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 21:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc (Another ferret session) > ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 31-May-11 12:19 yes? use "/tmp/OS_MBARI-M1_20101027_R_TS.nc" yes? list/l=1:10 temp VARIABLE : Hourly sea_water_temperature (celsius) FILENAME : OS_MBARI-M1_20101027_R_TS.nc FILEPATH : /tmp/ SUBSET : 11 by 10 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 27-OCT-2010 21:30 / 1: 14.09 13.60 .... .... .... .... .... .... .... .... .... 27-OCT-2010 22:30 / 2: 14.17 13.62 .... .... .... .... .... .... .... .... .... 27-OCT-2010 23:30 / 3: 14.34 13.63 .... .... .... .... .... .... .... .... .... 28-OCT-2010 00:30 / 4: 14.28 13.69 .... .... .... .... .... .... .... .... .... 28-OCT-2010 01:30 / 5: 14.22 13.72 .... .... .... .... .... .... .... .... .... 28-OCT-2010 02:30 / 6: 14.21 13.78 .... .... .... .... .... .... .... .... .... 28-OCT-2010 03:30 / 7: 14.21 13.76 .... .... .... .... .... .... .... .... .... 28-OCT-2010 04:30 / 8: 14.09 13.79 .... .... .... .... .... .... .... .... .... 28-OCT-2010 05:30 / 9: 14.01 13.77 .... .... .... .... .... .... .... .... .... 28-OCT-2010 06:30 / 10: 13.89 13.76 .... .... .... .... .... .... .... .... .... yes? {noformat} O.K.! That looks good. The 10m data got written. Now to fix it for production. This deployment was a special case where the 1m data start after the 10m data. Typically, it's most convenient to have all of the microcats have the same deployment start time. This makes the follow on data processing much simpler. All of the deployment start times in SSDS_METADTA for the M1 201010 deployment were adjusted to be the same as the start time for the Mooring deployment. Performing this step is now part of the standard operating procedure for perming mooring turns. Once SSDS_METADATA has been updated the individual instrument netCDF files must be created by DStoNetCDF.pl before combineTS.pl in run. There are still some problems with gappy data at 40m and below in the _TS file. To debug these we'll make another copy of the OS_MBARI-M1_20101027_R_TS.jnl to /tmp and edit it to go through the processing of the 40m data saving the data to a netCDF file in /tmp so as not to disturb the production processing. We also comment out all of the "CANCEL DATA 1" statements so that we can examine data from the files. This file is executed and we see no errors. We need to examine the input data and the Ferret variables that are constructed for the gridding. Here are the commands from the .jnl that perform the the gridding for the 40m data: {noformat} ! ! Remove temperature outliers before gridding ! LET Temperature_QFLAG = IF Temperature GT 2 AND Temperature LT 20 THEN 1 ELSE (-99999) SET VAR/BAD=-99999 Temperature_QFLAG LET Temperature_QC = Temperature_QFLAG * Temperature ! ! Compute my own mean of the data, making sure to assign missing values in the gaps in the Gap FLAG ! Allow at least 1 data point in each destination cell - there are usually 6 for 10 minute data ! LET Temperature_GOOD = Temperature_QC[gt=TIME@SUM] / Temperature_QC[gt=TIME@NGD] LET Temperature_GFLAG = IF Temperature[gt=TIME@NGD,gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] LT 1 THEN (-99999) ELSE 1 SET VAR/BAD=-99999 Temperature_GFLAG LET TEMP = Temperature_GFLAG * Temperature_GOOD[gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] {noformat} Here are some sample Ferret commands for plotting these data from the session that just executed the OS_MBARI-M1_20101027_R_TS.jnl script: {noformat} yes? go "OS_MBARI-M1_20101027_R_TS.jnl" yes? set region/t=1-may-2011:1-jun-2011 yes? plot Temperature_QC yes? plot/ov/symbol=1 temperature yes? set win 2 yes? plot Temperature_QC[gt=TIME@NGD] yes? plot TEMPERATURE_QC[GT=TIME@SUM] {noformat} ]]> 17236083 17268850 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the change log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is [http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log] # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: [http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED] To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:`PSAL,return=lend`/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC !-> LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:5182/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the first 3 temperature values from the 1m and 10m input CTD data and the first 3 times of the output axis: {noformat} yes? list/l=1:3/d=1 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1338 at original sampling intervals FILENAME : m1_ctd0001_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 1 122W 1 27-OCT-2010 21:08:43 / 1: 14.01 27-OCT-2010 21:22:27 / 2: 14.10 27-OCT-2010 21:28:23 / 3: 14.10 yes? list/l=1:3/d=2 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 27-OCT-2010 20:11:17 / 1: 13.38 27-OCT-2010 20:21:16 / 2: 13.59 27-OCT-2010 20:31:16 / 3: 13.45 yes? show/l=1:3 axis time name axis # pts start end TIME TIME 5181 r 27-OCT-2010 20:30 31-MAY-2011 16:30 T0 = 01-JAN-1950 00:00:00 Axis span (to cell edges) = 215.875 L T TBOX TBOXLO TSTEP (DAYS) 1> 27-OCT-2010 20:30:00 0.0416667 27-OCT-2010 20:00:00 22214.85 2> 27-OCT-2010 21:30:00 0.0416667 27-OCT-2010 21:00:00 22214.9 3> 27-OCT-2010 22:30:00 0.0416667 27-OCT-2010 22:00:00 22214.94 {noformat} The problem seems to be that the 10m data begin at 20:11:17 before the 1m data at 21:08:43. The first write of the 1m data to the output netCDF file gives the file its shape with the KLIMITS= and LLIMITS= options. Writing the 10m data, which begins in before the bounds that were defined violates Ferret's axis ordering logic. To test this hypothesis edit the SAVE statement in the temporary .jnl file to make the start time for the 10m data 21:00, execute it and then examine the output file: {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 21:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc (Another ferret session) > ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 31-May-11 12:19 yes? use "/tmp/OS_MBARI-M1_20101027_R_TS.nc" yes? list/l=1:10 temp VARIABLE : Hourly sea_water_temperature (celsius) FILENAME : OS_MBARI-M1_20101027_R_TS.nc FILEPATH : /tmp/ SUBSET : 11 by 10 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 27-OCT-2010 21:30 / 1: 14.09 13.60 .... .... .... .... .... .... .... .... .... 27-OCT-2010 22:30 / 2: 14.17 13.62 .... .... .... .... .... .... .... .... .... 27-OCT-2010 23:30 / 3: 14.34 13.63 .... .... .... .... .... .... .... .... .... 28-OCT-2010 00:30 / 4: 14.28 13.69 .... .... .... .... .... .... .... .... .... 28-OCT-2010 01:30 / 5: 14.22 13.72 .... .... .... .... .... .... .... .... .... 28-OCT-2010 02:30 / 6: 14.21 13.78 .... .... .... .... .... .... .... .... .... 28-OCT-2010 03:30 / 7: 14.21 13.76 .... .... .... .... .... .... .... .... .... 28-OCT-2010 04:30 / 8: 14.09 13.79 .... .... .... .... .... .... .... .... .... 28-OCT-2010 05:30 / 9: 14.01 13.77 .... .... .... .... .... .... .... .... .... 28-OCT-2010 06:30 / 10: 13.89 13.76 .... .... .... .... .... .... .... .... .... yes? {noformat} O.K.\! That looks good. The 10m data got written. Now to fix it for production. This deployment was a special case where the 1m data start after the 10m data. Typically, it's most convenient to have all of the microcats have the same deployment start time. This makes the follow on data processing much simpler. All of the deployment start times in SSDS_METADTA for the M1 201010 deployment were adjusted to be the same as the start time for the Mooring deployment. Performing this step is now part of the standard operating procedure for perming mooring turns. Once SSDS_METADATA has been updated the individual instrument netCDF files must be created by DStoNetCDF.pl before combineTS.pl in run. There are still some problems with gappy data at 40m and below in the \_TS file. To debug these we'll make another copy of the OS_MBARI-M1_20101027_R_TS.jnl to /tmp and edit it to go through the processing of the 40m data saving the data to a netCDF file in /tmp so as not to disturb the production processing. We also comment out all of the "CANCEL DATA 1" statements so that we can examine data from the files. This file is executed and we see no errors. We need to examine the input data and the Ferret variables that are constructed for the gridding. Here are the commands from the .jnl that perform the the gridding for the 40m data: {noformat} ! ! Remove temperature outliers before gridding ! LET Temperature_QFLAG = IF Temperature GT 2 AND Temperature LT 20 THEN 1 ELSE (-99999) SET VAR/BAD=-99999 Temperature_QFLAG LET Temperature_QC = Temperature_QFLAG * Temperature ! ! Compute my own mean of the data, making sure to assign missing values in the gaps in the Gap FLAG ! Allow at least 1 data point in each destination cell - there are usually 6 for 10 minute data ! LET Temperature_GOOD = Temperature_QC[gt=TIME@SUM] / Temperature_QC[gt=TIME@NGD] LET Temperature_GFLAG = IF Temperature[gt=TIME@NGD,gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] LT 1 THEN (-99999) ELSE 1 SET VAR/BAD=-99999 Temperature_GFLAG LET TEMP = Temperature_GFLAG * Temperature_GOOD[gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] {noformat} Here are some sample Ferret commands for plotting these data from the session that just executed the OS_MBARI-M1_20101027_R_TS.jnl script: {noformat} yes? go "OS_MBARI-M1_20101027_R_TS.jnl" yes? set region/t=1-may-2011:1-jun-2011 yes? plot Temperature_QC yes? plot/ov/symbol=1 temperature yes? set win 2 yes? plot Temperature_QC[gt=TIME@NGD] yes? plot TEMPERATURE_QC[GT=TIME@SUM] {noformat} These last two plot commands indicate the source of the problem of the gappy data from the inductive modem microcats. Another big clue is the comment in the .jnl file referring to the 10-minute input data and having at least 6 good values within each gridding cell. As can be seen in the plot we often have less than 1 for the @NGD value for the 40m data. These data are hourly and we should be using a different gridding algorithm for them. !m1_ngd_sum.gif|thumbnail!]]> 17236085 17268838 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the change log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:`PSAL,return=lend`/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC !-> LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:5182/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the first 3 temperature values from the 1m and 10m input CTD data and the first 3 times of the output axis: {noformat} yes? list/l=1:3/d=1 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1338 at original sampling intervals FILENAME : m1_ctd0001_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 1 122W 1 27-OCT-2010 21:08:43 / 1: 14.01 27-OCT-2010 21:22:27 / 2: 14.10 27-OCT-2010 21:28:23 / 3: 14.10 yes? list/l=1:3/d=2 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 27-OCT-2010 20:11:17 / 1: 13.38 27-OCT-2010 20:21:16 / 2: 13.59 27-OCT-2010 20:31:16 / 3: 13.45 yes? show/l=1:3 axis time name axis # pts start end TIME TIME 5181 r 27-OCT-2010 20:30 31-MAY-2011 16:30 T0 = 01-JAN-1950 00:00:00 Axis span (to cell edges) = 215.875 L T TBOX TBOXLO TSTEP (DAYS) 1> 27-OCT-2010 20:30:00 0.0416667 27-OCT-2010 20:00:00 22214.85 2> 27-OCT-2010 21:30:00 0.0416667 27-OCT-2010 21:00:00 22214.9 3> 27-OCT-2010 22:30:00 0.0416667 27-OCT-2010 22:00:00 22214.94 {noformat} The problem seems to be that the 10m data begin at 20:11:17 before the 1m data at 21:08:43. The first write of the 1m data to the output netCDF file gives the file its shape with the KLIMITS= and LLIMITS= options. Writing the 10m data, which begins in before the bounds that were defined violates Ferret's axis ordering logic. To test this hypothesis edit the SAVE statement in the temporary .jnl file to make the start time for the 10m data 21:00, execute it and then examine the output file: {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 21:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc (Another ferret session) > ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 31-May-11 12:19 yes? use "/tmp/OS_MBARI-M1_20101027_R_TS.nc" yes? list/l=1:10 temp VARIABLE : Hourly sea_water_temperature (celsius) FILENAME : OS_MBARI-M1_20101027_R_TS.nc FILEPATH : /tmp/ SUBSET : 11 by 10 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 27-OCT-2010 21:30 / 1: 14.09 13.60 .... .... .... .... .... .... .... .... .... 27-OCT-2010 22:30 / 2: 14.17 13.62 .... .... .... .... .... .... .... .... .... 27-OCT-2010 23:30 / 3: 14.34 13.63 .... .... .... .... .... .... .... .... .... 28-OCT-2010 00:30 / 4: 14.28 13.69 .... .... .... .... .... .... .... .... .... 28-OCT-2010 01:30 / 5: 14.22 13.72 .... .... .... .... .... .... .... .... .... 28-OCT-2010 02:30 / 6: 14.21 13.78 .... .... .... .... .... .... .... .... .... 28-OCT-2010 03:30 / 7: 14.21 13.76 .... .... .... .... .... .... .... .... .... 28-OCT-2010 04:30 / 8: 14.09 13.79 .... .... .... .... .... .... .... .... .... 28-OCT-2010 05:30 / 9: 14.01 13.77 .... .... .... .... .... .... .... .... .... 28-OCT-2010 06:30 / 10: 13.89 13.76 .... .... .... .... .... .... .... .... .... yes? {noformat} O.K.! That looks good. The 10m data got written. Now to fix it for production. This deployment was a special case where the 1m data start after the 10m data. Typically, it's most convenient to have all of the microcats have the same deployment start time. This makes the follow on data processing much simpler. ]]> 17236073 10388736 New->Other...' and then select 'Server->Server'. Then 'Next>' # Then select 'JBoss->JBoss 4.0'. The default server host of 'localhost' and server name of 'JBoss v4.0 on localhost' is fine. Click on 'Next>'. # You can use the Default JRE and for the 'Application Server Directory' browse to the location where you unpacked JBoss.]]> 10356010 17268804 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:`PSAL,return=lend`/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC !-> LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:5182/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the first 3 temperature values from the 1m and 10m input CTD data and the first 3 times of the output axis: {noformat} yes? list/l=1:3/d=1 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1338 at original sampling intervals FILENAME : m1_ctd0001_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 1 122W 1 27-OCT-2010 21:08:43 / 1: 14.01 27-OCT-2010 21:22:27 / 2: 14.10 27-OCT-2010 21:28:23 / 3: 14.10 yes? list/l=1:3/d=2 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 27-OCT-2010 20:11:17 / 1: 13.38 27-OCT-2010 20:21:16 / 2: 13.59 27-OCT-2010 20:31:16 / 3: 13.45 yes? show/l=1:3 axis time name axis # pts start end TIME TIME 5181 r 27-OCT-2010 20:30 31-MAY-2011 16:30 T0 = 01-JAN-1950 00:00:00 Axis span (to cell edges) = 215.875 L T TBOX TBOXLO TSTEP (DAYS) 1> 27-OCT-2010 20:30:00 0.0416667 27-OCT-2010 20:00:00 22214.85 2> 27-OCT-2010 21:30:00 0.0416667 27-OCT-2010 21:00:00 22214.9 3> 27-OCT-2010 22:30:00 0.0416667 27-OCT-2010 22:00:00 22214.94 {noformat} The problem seems to be that the 10m data begin at 20:11:17 before the 1m data at 21:08:43. The first write of the 1m data to the output netCDF file gives the file its shape with the KLIMITS= and LLIMITS= options. Writing the 10m data, which begins in before the bounds that were defined violates Ferret's axis ordering logic. To test this hypothesis edit the SAVE statement in the temporary .jnl file to make the start time for the 10m data 21:00, execute it and then examine the output file: {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 21:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc (Another ferret session) > ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 31-May-11 12:19 yes? use "/tmp/OS_MBARI-M1_20101027_R_TS.nc" yes? list/l=1:10 temp VARIABLE : Hourly sea_water_temperature (celsius) FILENAME : OS_MBARI-M1_20101027_R_TS.nc FILEPATH : /tmp/ SUBSET : 11 by 10 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 27-OCT-2010 21:30 / 1: 14.09 13.60 .... .... .... .... .... .... .... .... .... 27-OCT-2010 22:30 / 2: 14.17 13.62 .... .... .... .... .... .... .... .... .... 27-OCT-2010 23:30 / 3: 14.34 13.63 .... .... .... .... .... .... .... .... .... 28-OCT-2010 00:30 / 4: 14.28 13.69 .... .... .... .... .... .... .... .... .... 28-OCT-2010 01:30 / 5: 14.22 13.72 .... .... .... .... .... .... .... .... .... 28-OCT-2010 02:30 / 6: 14.21 13.78 .... .... .... .... .... .... .... .... .... 28-OCT-2010 03:30 / 7: 14.21 13.76 .... .... .... .... .... .... .... .... .... 28-OCT-2010 04:30 / 8: 14.09 13.79 .... .... .... .... .... .... .... .... .... 28-OCT-2010 05:30 / 9: 14.01 13.77 .... .... .... .... .... .... .... .... .... 28-OCT-2010 06:30 / 10: 13.89 13.76 .... .... .... .... .... .... .... .... .... yes? {noformat} O.K.! That looks good. The 10m data got written. Now to fix it for production. This deployment was a special case where the 1m data start after the 10m data. Typically, it's most convenient to have all of the microcats have the same deployment start time. This makes the follow on data processing much simpler. ]]> 17236038 17268803 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat} To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:`PSAL,return=lend`/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC !-> LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:5182/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the first 3 temperature values from the 1m and 10m input CTD data and the first 3 times of the output axis: {noformat} yes? list/l=1:3/d=1 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1338 at original sampling intervals FILENAME : m1_ctd0001_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 1 122W 1 27-OCT-2010 21:08:43 / 1: 14.01 27-OCT-2010 21:22:27 / 2: 14.10 27-OCT-2010 21:28:23 / 3: 14.10 yes? list/l=1:3/d=2 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 27-OCT-2010 20:11:17 / 1: 13.38 27-OCT-2010 20:21:16 / 2: 13.59 27-OCT-2010 20:31:16 / 3: 13.45 yes? show/l=1:3 axis time name axis # pts start end TIME TIME 5181 r 27-OCT-2010 20:30 31-MAY-2011 16:30 T0 = 01-JAN-1950 00:00:00 Axis span (to cell edges) = 215.875 L T TBOX TBOXLO TSTEP (DAYS) 1> 27-OCT-2010 20:30:00 0.0416667 27-OCT-2010 20:00:00 22214.85 2> 27-OCT-2010 21:30:00 0.0416667 27-OCT-2010 21:00:00 22214.9 3> 27-OCT-2010 22:30:00 0.0416667 27-OCT-2010 22:00:00 22214.94 {noformat} The problem seems to be that the 10m data begin at 20:11:17 before the 1m data at 21:08:43. The first write of the 1m data to the output netCDF file gives the file its shape with the KLIMITS= and LLIMITS= options. Writing the 10m data, which begins in before the bounds that were defined violates Ferret's axis ordering logic.]]> 17236037 8388717 8355953 25428137 25395369 4456499 4489265 2 2006-10-17 21:39:33.420 2006-10-18 08:17:38.940 9 7798888 tag. For instance any nominalLat/Lon/Depth. {color:#ff0000}The one exception is the nominalDepth, if it is known please set it{color}. # If the tag has a 'name' attribute, make sure it does not have any deployment specific information in it. For example, 'ISUS Deployment' is better than 'M2 ISUS Deployment'. The reason for removing any deployment information from the XML is so that when the device moves to a different mooring, the user should not have to edit the XML. The goal is to get all the XML to a point where it never needs to be edited when an instrument is deployed (unless something in the way the data stream is generated from the instrument changes). # Go to the SSDS Device pages and verify that the all the device information (mfg, model, serial number, name, type, etc.) matches what is currently in SSDS. If any of those are different it will update the device information in SSDS when the XML comes in the data stream. # Verify RecordDescription and RecordVariables look correct. I usually go to the raw data pages in SSDS and bring up the last few packets from the device just to verify that the number of columns and bufferSeparator look about right. # Check any changes to the XML back into CVS. # Copy the XML to the \\Tornado\ssdsdata\mooring(m1\|m2)\YYYY\xml directory # Go to the \\Tornado\ssdsdata\mooring(m1\|m2)\YYYY\cfg directory. # This next step is the one that needs to be timed with the mooring turn. When the old instrument is shutdown: ## Open the ssds.cfg file in a text editor ## Find the line that shows the currently deployed instrument and copy it to a line just below it. For example, if we are replacing the GPS, it might look like this before: {panel:title=Before Copy} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} and this after: {panel:title=After Copy} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # Now change the new line to have the correct device ID and the correct XML file URL {panel:title=After Device ID update} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = GPS_TYPE3,1511,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1511.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # To clean up the previous deployment information, put start and end dates after the XML URL {panel:title=After Adding Start/End dates} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,2007/04/25 16:21:58,2007/08/01 10:00:00,TransformGPS instrument = GPS_TYPE3,1511,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1511.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} (Be careful on the format for the start and end date strings.  Leading 0s are required.) # Save the cfg file. {note:title=Saving the file will make the change take hold}When the ssds.cfg file changes (saved) is when the OASIS2SSDS processing will pick up the instrument change. Now, the next time it runs it will pick up the instrument change, grab the XML file from the 'xml' directory and publish it to SSDS. It will then publish all data under the new device ID. {note} # Edit the metadata in SSDS to put a close date on the old instrument deployment in SSDS. Currently I do that using Enterprise Manager. h5. If this is being done after the fact the next steps will also need to be taken. # After the new deployment shows up in SSDS (which can take up to 5 minutes after the OASIS2SSDS has completed), the start time for the new deployment will need to be edited to match the actual time the instrument was swapped. # Also because the data was being published under the incorrect device ID, it will need to be moved from one database table in SSDS_Data on Solstice to another table. ## The first thing that I do is grab the timestamp from the last packet sent from the old device in the raw data page on SSDS. ## For example, I go to: [http://new-ssds.mbari.org:8080/ssds/siamRawDataStep1.jsp] and enter the old device ID and set the number of packets back to make sure it goes far enough back to cover the actual time of the instrument swap. Then click on 'Next->'. ## Once the raw data shows up, find the last packet from the old device and grab the 'SIAM Timestamp' value (not the date/time) as that will be used in the Enterprise Manager query. ## Open Enterprise Manager and navigate to the 'SSDS_Data' database on Solstice. ## Browse the tables and find the table with the device ID of the old device and right click on it and select 'Open Table->Return all rows'. ## Click on the 'SQL' button in Enterprise Manager to bring up the SQL pane. It should show the basic query which should look something like this: {noformat} SELECT * FROM [1416] {noformat} ## Now add the where clause to pick only the data that is after the timestamp you grabbed from the last packet on the web page. {note:title=Timestamps in SQL are in Seconds}A quick note here, the 'SIAM Timestamp' on the raw data page is actually in milliseconds and the database column is in seconds so you will have to remove the last three digits of the 'SIAM Timestamp' before putting it in this query. {note} {noformat} SELECT * FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ## Run this query by clicking the run button '\!' in Enterprise Manager. ## Look over the results to make sure they look about right (usually you are looking for the length of the return which should be much shorter). You can actually use a count query to see how many rows this query will return. A count query would look like: {noformat} SELECT count(*) FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ## Once you know the query is correct (also compare sequence number in query return and the raw data page), copy it to the clipboard and close the query window in Enterprise Manager. ## Navigate to the table of the device you want to copy the data into and right click and select 'All Tasks->Import Data...' which will fire up the DTS wizard. ### Click on 'Next>' ### For the Data Source database choose Solstice ### Select the 'SSDS_Data' database (note you should have permissions to do all this and use your windows authentication) ### Click on 'Next>' ### The destination configuration should be already to go (Solistice and SSDS_Data database). ### Click on 'Next>' ### Select 'Use a query to specify the data to transfer' ### Click on 'Next>' ### Paste the query from your clipboard into the 'Query Statement' window (You can click on 'Parse' if you want a quick sanity check) ### Click on 'Next>' ### Click on the 'Results' entry under the 'Destination' column which will enable a drop down box. ### Choose the table of the newly installed device where you will be copying the data to. ### Click on 'Next>' ### Click on 'Next>' ### Click on 'Finish' which will copy the data. ### Once that is done, open the table of the old instrument and the SQL pane so that we can construct the delete query on the old data. ### Paste in the select query and verify it is the same data you copied over: {noformat} SELECT * FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ### If it looks good, click on the 'Change Query type ...' button in Enterprise Manager and select 'Delete'. This will change the query to a delete query. ### Run the query by click on the run '\!' button. That will remove all the data from the old instrument. That's it ... whew\! Kevin Gomes (August 3, 2007)]]> 7766124 7798884 tag. For instance any nominalLat/Lon/Depth. {color:#ff0000}The one exception is the nominalDepth, if it is known please set it{color}. # If the tag has a 'name' attribute, make sure it does not have any deployment specific information in it. For example, 'ISUS Deployment' is better than 'M2 ISUS Deployment'. The reason for removing any deployment information from the XML is so that when the device moves to a different mooring, the user should not have to edit the XML. The goal is to get all the XML to a point where it never needs to be edited when an instrument is deployed (unless something in the way the data stream is generated from the instrument changes). # Go to the SSDS Device pages and verify that the all the device information (mfg, model, serial number, name, type, etc.) matches what is currently in SSDS. If any of those are different it will update the device information in SSDS when the XML comes in the data stream. # Verify RecordDescription and RecordVariables look correct. I usually go to the raw data pages in SSDS and bring up the last few packets from the device just to verify that the number of columns and bufferSeparator look about right. # Check any changes to the XML back into CVS. # Copy the XML to the \\Tornado\ssdsdata\mooring(m1\|m2)\YYYY\xml directory # Go to the \\Tornado\ssdsdata\mooring(m1\|m2)\YYYY\cfg directory. # This next step is the one that needs to be timed with the mooring turn. When the old instrument is shutdown: ## Open the ssds.cfg file in a text editor ## Find the line that shows the currently deployed instrument and copy it to a line just below it. For example, if we are replacing the GPS, it might look like this before: {panel:title=Before Copy} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} and this after: {panel:title=After Copy} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # Now change the new line to have the correct device ID and the correct XML file URL {panel:title=After Device ID update} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = GPS_TYPE3,1511,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1511.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # To clean up the previous deployment information, put start and end dates after the XML URL {panel:title=After Adding Start/End dates} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,2007/04/25 16:21:58,2007/08/01 10:00:00,TransformGPS instrument = GPS_TYPE3,1511,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1511.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} (Be careful on the format for the start and end date strings.  Leading 0s are required.) # Save the cfg file. {note:title=Saving the file will make the change take hold}When the ssds.cfg file changes (saved) is when the OASIS2SSDS processing will pick up the instrument change. Now, the next time it runs it will pick up the instrument change, grab the XML file from the 'xml' directory and publish it to SSDS. It will then publish all data under the new device ID. {note} # Edit the metadata in SSDS to put a close date on the old instrument deployment in SSDS. Currently I do that using Enterprise Manager. h5. If this is being done after the fact the next steps will also need to be taken. # After the new deployment shows up in SSDS, the start time for the new deployment will need to be edited to match the actual time the instrument was swapped. # Also because the data was being published under the incorrect device ID, it will need to be moved from one database table in SSDS_Data on Solstice to another table. ## The first thing that I do is grab the timestamp from the last packet sent from the old device in the raw data page on SSDS. ## For example, I go to: [http://new-ssds.mbari.org:8080/ssds/siamRawDataStep1.jsp] and enter the old device ID and set the number of packets back to make sure it goes far enough back to cover the actual time of the instrument swap. Then click on 'Next->'. ## Once the raw data shows up, find the last packet from the old device and grab the 'SIAM Timestamp' value (not the date/time) as that will be used in the Enterprise Manager query. ## Open Enterprise Manager and navigate to the 'SSDS_Data' database on Solstice. ## Browse the tables and find the table with the device ID of the old device and right click on it and select 'Open Table->Return all rows'. ## Click on the 'SQL' button in Enterprise Manager to bring up the SQL pane. It should show the basic query which should look something like this: {noformat} SELECT * FROM [1416] {noformat} ## Now add the where clause to pick only the data that is after the timestamp you grabbed from the last packet on the web page. {note:title=Timestamps in SQL are in Seconds}A quick note here, the 'SIAM Timestamp' on the raw data page is actually in milliseconds and the database column is in seconds so you will have to remove the last three digits of the 'SIAM Timestamp' before putting it in this query. {note} {noformat} SELECT * FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ## Run this query by clicking the run button '\!' in Enterprise Manager. ## Look over the results to make sure they look about right (usually you are looking for the length of the return which should be much shorter). You can actually use a count query to see how many rows this query will return. A count query would look like: {noformat} SELECT count(*) FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ## Once you know the query is correct, copy it to the clipboard and close the query window in Enterprise Manager. ## Navigate to the table of the device you want to copy the data into and right click and select 'All Tasks->Import Data...' which will fire up the DTS wizard. ### Click on 'Next>' ### For the Data Source database choose Solstice ### Select the 'SSDS_Data' database (note you should have permissions to do all this and use your windows authentication) ### Click on 'Next>' ### The destination configuration should be already to go (Solistice and SSDS_Data database). ### Click on 'Next>' ### Select 'Use a query to specify the data to transfer' ### Click on 'Next>' ### Paste the query from your clipboard into the 'Query Statement' window (You can click on 'Parse' if you want a quick sanity check) ### Click on 'Next>' ### Click on the 'Results' entry under the 'Destination' column which will enable a drop down box. ### Choose the table of the newly installed device where you will be copying the data to. ### Click on 'Next>' ### Click on 'Next>' ### Click on 'Finish' which will copy the data. ### Once that is done, open the table of the old instrument and the SQL pane so that we can construct the delete query on the old data. ### Paste in the select query and verify it is the same data you copied over: {noformat} SELECT * FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ### If it looks good, click on the 'Change Query type ...' button in Enterprise Manager and select 'Delete'. This will change the query to a delete query. ### Run the query by click on the run '\!' button. That will remove all the data from the old instrument. That's it ... whew\! Kevin Gomes (August 3, 2007)]]> 7766120 7798886 tag. For instance any nominalLat/Lon/Depth. {color:#ff0000}The one exception is the nominalDepth, if it is known please set it{color}. # If the tag has a 'name' attribute, make sure it does not have any deployment specific information in it. For example, 'ISUS Deployment' is better than 'M2 ISUS Deployment'. The reason for removing any deployment information from the XML is so that when the device moves to a different mooring, the user should not have to edit the XML. The goal is to get all the XML to a point where it never needs to be edited when an instrument is deployed (unless something in the way the data stream is generated from the instrument changes). # Go to the SSDS Device pages and verify that the all the device information (mfg, model, serial number, name, type, etc.) matches what is currently in SSDS. If any of those are different it will update the device information in SSDS when the XML comes in the data stream. # Verify RecordDescription and RecordVariables look correct. I usually go to the raw data pages in SSDS and bring up the last few packets from the device just to verify that the number of columns and bufferSeparator look about right. # Check any changes to the XML back into CVS. # Copy the XML to the \\Tornado\ssdsdata\mooring(m1\|m2)\YYYY\xml directory # Go to the \\Tornado\ssdsdata\mooring(m1\|m2)\YYYY\cfg directory. # This next step is the one that needs to be timed with the mooring turn. When the old instrument is shutdown: ## Open the ssds.cfg file in a text editor ## Find the line that shows the currently deployed instrument and copy it to a line just below it. For example, if we are replacing the GPS, it might look like this before: {panel:title=Before Copy} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} and this after: {panel:title=After Copy} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # Now change the new line to have the correct device ID and the correct XML file URL {panel:title=After Device ID update} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = GPS_TYPE3,1511,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1511.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # To clean up the previous deployment information, put start and end dates after the XML URL {panel:title=After Adding Start/End dates} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,2007/04/25 16:21:58,2007/08/01 10:00:00,TransformGPS instrument = GPS_TYPE3,1511,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1511.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} (Be careful on the format for the start and end date strings.  Leading 0s are required.) # Save the cfg file. {note:title=Saving the file will make the change take hold}When the ssds.cfg file changes (saved) is when the OASIS2SSDS processing will pick up the instrument change. Now, the next time it runs it will pick up the instrument change, grab the XML file from the 'xml' directory and publish it to SSDS. It will then publish all data under the new device ID. {note} # Edit the metadata in SSDS to put a close date on the old instrument deployment in SSDS. Currently I do that using Enterprise Manager. h5. If this is being done after the fact the next steps will also need to be taken. # After the new deployment shows up in SSDS (which can take up to 5 minutes after the OASIS2SSDS has completed), the start time for the new deployment will need to be edited to match the actual time the instrument was swapped. # Also because the data was being published under the incorrect device ID, it will need to be moved from one database table in SSDS_Data on Solstice to another table. ## The first thing that I do is grab the timestamp from the last packet sent from the old device in the raw data page on SSDS. ## For example, I go to: [http://new-ssds.mbari.org:8080/ssds/siamRawDataStep1.jsp] and enter the old device ID and set the number of packets back to make sure it goes far enough back to cover the actual time of the instrument swap. Then click on 'Next->'. ## Once the raw data shows up, find the last packet from the old device and grab the 'SIAM Timestamp' value (not the date/time) as that will be used in the Enterprise Manager query. ## Open Enterprise Manager and navigate to the 'SSDS_Data' database on Solstice. ## Browse the tables and find the table with the device ID of the old device and right click on it and select 'Open Table->Return all rows'. ## Click on the 'SQL' button in Enterprise Manager to bring up the SQL pane. It should show the basic query which should look something like this: {noformat} SELECT * FROM [1416] {noformat} ## Now add the where clause to pick only the data that is after the timestamp you grabbed from the last packet on the web page. {note:title=Timestamps in SQL are in Seconds}A quick note here, the 'SIAM Timestamp' on the raw data page is actually in milliseconds and the database column is in seconds so you will have to remove the last three digits of the 'SIAM Timestamp' before putting it in this query. {note} {noformat} SELECT * FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ## Run this query by clicking the run button '\!' in Enterprise Manager. ## Look over the results to make sure they look about right (usually you are looking for the length of the return which should be much shorter). You can actually use a count query to see how many rows this query will return. A count query would look like: {noformat} SELECT count(*) FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ## Once you know the query is correct, copy it to the clipboard and close the query window in Enterprise Manager. ## Navigate to the table of the device you want to copy the data into and right click and select 'All Tasks->Import Data...' which will fire up the DTS wizard. ### Click on 'Next>' ### For the Data Source database choose Solstice ### Select the 'SSDS_Data' database (note you should have permissions to do all this and use your windows authentication) ### Click on 'Next>' ### The destination configuration should be already to go (Solistice and SSDS_Data database). ### Click on 'Next>' ### Select 'Use a query to specify the data to transfer' ### Click on 'Next>' ### Paste the query from your clipboard into the 'Query Statement' window (You can click on 'Parse' if you want a quick sanity check) ### Click on 'Next>' ### Click on the 'Results' entry under the 'Destination' column which will enable a drop down box. ### Choose the table of the newly installed device where you will be copying the data to. ### Click on 'Next>' ### Click on 'Next>' ### Click on 'Finish' which will copy the data. ### Once that is done, open the table of the old instrument and the SQL pane so that we can construct the delete query on the old data. ### Paste in the select query and verify it is the same data you copied over: {noformat} SELECT * FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ### If it looks good, click on the 'Change Query type ...' button in Enterprise Manager and select 'Delete'. This will change the query to a delete query. ### Run the query by click on the run '\!' button. That will remove all the data from the old instrument. That's it ... whew\! Kevin Gomes (August 3, 2007)]]> 7766122 7798882 tag. For instance any nominalLat/Lon/Depth. # If the tag has a 'name' attribute, make sure it does not have any deployment specific information in it. For example, 'ISUS Deployment' is better than 'M2 ISUS Deployment'. The reason for removing any deployment information from the XML is so that when the device moves to a different mooring, the user should not have to edit the XML. The goal is to get all the XML to a point where it never needs to be edited when an instrument is deployed (unless something in the way the data stream is generated from the instrument changes). # Go to the SSDS Device pages and verify that the all the device information (mfg, model, serial number, name, type, etc.) matches what is currently in SSDS. If any of those are different it will update the device information in SSDS when the XML comes in the data stream. # Verify RecordDescription and RecordVariables look correct. I usually go to the raw data pages in SSDS and bring up the last few packets from the device just to verify that the number of columns and bufferSeparator look about right. # Check any changes to the XML back into CVS. # Copy the XML to the \\Tornado\ssdsdata\mooring(m1\|m2)\YYYY\xml directory # Go to the \\Tornado\ssdsdata\mooring(m1\|m2)\YYYY\cfg directory. # This next step is the one that needs to be timed with the mooring turn. When the old instrument is shutdown: ## Open the ssds.cfg file in a text editor ## Find the line that shows the currently deployed instrument and copy it to a line just below it. For example, if we are replacing the GPS, it might look like this before: {panel:title=Before Copy} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} and this after: {panel:title=After Copy} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # Now change the new line to have the correct device ID and the correct XML file URL {panel:title=After Device ID update} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = GPS_TYPE3,1511,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1511.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # To clean up the previous deployment information, put start and end dates after the XML URL {panel:title=After Adding Start/End dates} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,2007/04/25 16:21:58,2007/08/01 10:00:00,TransformGPS instrument = GPS_TYPE3,1511,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1511.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} (Be careful on the format for the start and end date strings.  Leading 0s are required.) # Save the cfg file. {note:title=Saving the file will make the change take hold}When the ssds.cfg file changes (saved) is when the OASIS2SSDS processing will pick up the instrument change. Now, the next time it runs it will pick up the instrument change, grab the XML file from the 'xml' directory and publish it to SSDS. It will then publish all data under the new device ID. {note} # Edit the metadata in SSDS to put a close date on the old instrument deployment in SSDS. Currently I do that using Enterprise Manager. h5. If this is being done after the fact the next steps will also need to be taken. # After the new deployment shows up in SSDS, the start time for the new deployment will need to be edited to match the actual time the instrument was swapped. # Also because the data was being published under the incorrect device ID, it will need to be moved from one database table in SSDS_Data on Solstice to another table. ## The first thing that I do is grab the timestamp from the last packet sent from the old device in the raw data page on SSDS. ## For example, I go to: [http://new-ssds.mbari.org:8080/ssds/siamRawDataStep1.jsp] and enter the old device ID and set the number of packets back to make sure it goes far enough back to cover the actual time of the instrument swap. Then click on 'Next->'. ## Once the raw data shows up, find the last packet from the old device and grab the 'SIAM Timestamp' value (not the date/time) as that will be used in the Enterprise Manager query. ## Open Enterprise Manager and navigate to the 'SSDS_Data' database on Solstice. ## Browse the tables and find the table with the device ID of the old device and right click on it and select 'Open Table->Return all rows'. ## Click on the 'SQL' button in Enterprise Manager to bring up the SQL pane. It should show the basic query which should look something like this: {noformat} SELECT * FROM [1416] {noformat} ## Now add the where clause to pick only the data that is after the timestamp you grabbed from the last packet on the web page. {note:title=Timestamps in SQL are in Seconds}A quick note here, the 'SIAM Timestamp' on the raw data page is actually in milliseconds and the database column is in seconds so you will have to remove the last three digits of the 'SIAM Timestamp' before putting it in this query. {note} {noformat} SELECT * FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ## Run this query by clicking the run button '\!' in Enterprise Manager. ## Look over the results to make sure they look about right (usually you are looking for the length of the return which should be much shorter). You can actually use a count query to see how many rows this query will return. A count query would look like: {noformat} SELECT count(*) FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ## Once you know the query is correct, copy it to the clipboard and close the query window in Enterprise Manager. ## Navigate to the table of the device you want to copy the data into and right click and select 'All Tasks->Import Data...' which will fire up the DTS wizard. ### Click on 'Next>' ### For the Data Source database choose Solstice ### Select the 'SSDS_Data' database (note you should have permissions to do all this and use your windows authentication) ### Click on 'Next>' ### The destination configuration should be already to go (Solistice and SSDS_Data database). ### Click on 'Next>' ### Select 'Use a query to specify the data to transfer' ### Click on 'Next>' ### Paste the query from your clipboard into the 'Query Statement' window (You can click on 'Parse' if you want a quick sanity check) ### Click on 'Next>' ### Click on the 'Results' entry under the 'Destination' column which will enable a drop down box. ### Choose the table of the newly installed device where you will be copying the data to. ### Click on 'Next>' ### Click on 'Next>' ### Click on 'Finish' which will copy the data. ### Once that is done, open the table of the old instrument and the SQL pane so that we can construct the delete query on the old data. ### Paste in the select query and verify it is the same data you copied over: {noformat} SELECT * FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ### If it looks good, click on the 'Change Query type ...' button in Enterprise Manager and select 'Delete'. This will change the query to a delete query. ### Run the query by click on the run '\!' button. That will remove all the data from the old instrument. That's it ... whew\! Kevin Gomes (August 3, 2007)]]> 7766118 7798880 7766116 7798878 7766114 7798877 7766113 7798863 7766099 7798859 7766095 7798858 7766094 8388871 0, the service will consider any time between // samples that is less than the specified gap plus this margin, // it will assume that it is not a gap condition. This is to // prevent false positives when the sample timestamps aren't exactly // on the interval. String gapSpec, // There are two ways to specify a gap: // SERVICE_CALCULATED or USER_SPECIFIED Long numberOfRecords, // If the call specifies SERVICE_CALCULATED and this // is greater than 0, the service will use 'numberOfRecords' // most recent records of the specified RecordType // in calculating the average time between samples Date intervalCalcStartWindow, // This is the date that starts the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfPoints is specified, this is ignored. Date intervalCalcEndWindow, // This is the date that ends the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this will be used as // the endtime and then the service will use the numberOfRecords // before this time as the data to calculate the average // time interval. Long gapInMillis // If the gapSpec is USER_SPECIFIED, then the service will use // this number of milliseconds as the gap for identifying gaps. ) {code} With a return that has the format of: Properties Objects with properties: ||Property Name||Value|| |lastPacketDateTime|This is the date and time of the last packet received| |totalNumberOfRecords|This is the total number of records for the parameters specified| |numberOfFuturePackets|This is the number of packet that appear in the future. This should be zero and if they are not, there could be bad data| |averagSampleIntervalInMillis|This is the number of milliseconds that the service used to find data gaps| |marginInMillis|This is the number of milliseconds as a margin that the service used to find data gaps| |dataGap1Start|This is the date and time of start of the first possible gap in the data| |dataGap1End|This is the date and time of end of the first possible gap in the data| |.|.| |.|.| |.|.| |dataGapNStart|This is the date and time of start of the Nth possible gap in the data| |dataGapNEnd|This is the date and time of end of the Nth possible gap in the data| h5. Java EJB client h5. REST client If you want to use the REST-style interface the HTTP call would looks something like: {code} http://localhost:8080/servlet/DataAccessServlet?objectToInvokeOn=SQLDataStreamRawDataAccess&method=getDataStreamProperties &p1Type=String&p1Value=1300 &p2Type=Long&p2Value=1 &p3Type=String&p3Value=10 &p4Type=Boolean&p4Value=false &p5Type=String&p5Value=0 {code} And the return might look something like {code} java.util.Properties{} {code} h5. Web Service Client {panel}]]> 8356114 8388873 0, the service will consider any time between // samples that is less than the specified gap plus this margin, // it will assume that it is not a gap condition. This is to // prevent false positives when the sample timestamps aren't exactly // on the interval. String gapSpec, // There are two ways to specify a gap: // SERVICE_CALCULATED or USER_SPECIFIED Long numberOfRecords, // If the call specifies SERVICE_CALCULATED and this // is greater than 0, the service will use 'numberOfRecords' // most recent records of the specified RecordType // in calculating the average time between samples Date intervalCalcStartWindow, // This is the date that starts the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfPoints is specified, this is ignored. Date intervalCalcEndWindow, // This is the date that ends the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this will be used as // the endtime and then the service will use the numberOfRecords // before this time as the data to calculate the average // time interval. Long gapInMillis // If the gapSpec is USER_SPECIFIED, then the service will use // this number of milliseconds as the gap for identifying gaps. ) {code} With a return that has the format of: Properties Objects with properties: ||Property Name||Value|| |lastPacketDateTime|This is the date and time of the last packet received| |totalNumberOfRecords|This is the total number of records for the parameters specified| |numberOfFuturePackets|This is the number of packet that appear in the future. This should be zero and if they are not, there could be bad data| |averagSampleIntervalInMillis|This is the number of milliseconds that the service used to find data gaps| |marginInMillis|This is the number of milliseconds as a margin that the service used to find data gaps| |dataGap1Start|This is the date and time of start of the first possible gap in the data| |dataGap1End|This is the date and time of end of the first possible gap in the data| |.|.| |.|.| |.|.| |dataGapNStart|This is the date and time of start of the Nth possible gap in the data| |dataGapNEnd|This is the date and time of end of the Nth possible gap in the data| h5. Java EJB client h5. REST client If you want to use the REST-style interface the HTTP call would looks something like: {code} http://localhost:8080/servlet/DataAccessServlet?objectToInvokeOn=SQLDataStreamRawDataAccess&method=getDataStreamProperties &p1Type=String&p1Value=1300 &p2Type=Long&p2Value=1 &p3Type=String&p3Value=10 &p4Type=Boolean&p4Value=false &p5Type=String&p5Value=0 {code} And the return might look something like {code} java.util.Properties{} {code} h5. Web Service Client {panel}]]> 8356116 3670677 3637927 15663510 15630753 15663512 15630755 3670680 3637930 15663514 15630757 15663515 15630758 3670685 3637936 15663518 15630761 3670683 3637934 15663520 15630763 8388892 8356136 3670689 3637940 3670687 3637938 3670691 3637942 8388905 8356152 3670693 3637944 3670695 3637946 3670698 3637949 8388914 {code} # It's recommended that the final XML is to be checked into the puckxml module in MBARI's CVS on moonjelly. # Submit the Metadata to SSDS using the SSDSLoads application (available at [http://new-ssds.mbari.org/ssds-docs/client/]. The \-h option provides a usage note): {code} java -jar ssdsLoads-new-ssds.jar -d 1696.xml {code} ## If a mistake is made in the metadata (e.g. forgetting the platform Deployment) it may be easier to undo the submission and start over - do a deep delete on the parent deployment if this is the case. ## Minor attribute fixes may be done by editing the database once the deployments have been loaded # Check that the metadata has been successfully loaded using the Explorer application. h2. Establish data publishing application # This step assumes that there will be some application reading data from the deployed instrument. Perhaps it is a legacy application which reads the data to load into a custom data storage or visualization system. For this application to publish data to SSDS it must have visibility of each record the instrument produces as that record needs to be "packaged" into a SensorDataPacket and "published" to SSDS. # An example Java application that will package and publish a record is below: {code} import java.io.IOException; import moos.ssds.jms.PublisherComponent; import moos.ssds.transmogrify.SSDSDevicePacket; /** *

* Publish instrument data records to the SSDS database. The client must provide * SSDS device ID, timeStamp, sequence number, and payload. *

*
* * @author : $Author: mccann $ * @version : $Revision: 1.17.2.7 $ *
*

* The * Monterey Bay Aquarium Research Institute (MBARI) provides this * documentation and code "as is", with no warranty, express * or implied, of its quality or consistency. It is provided without * support and without obligation on the part of MBARI to assist in its * use, correction, modification, or enhancement. This information * should not be published or distributed to third parties without * specific written permission from MBARI. *

*
* Copyright 2008 MBARI.
* MBARI Proprietary Information. All rights reserved.
*
*
*/ /** * @author mccann * */ public class SsdsPublisher { /** * Publish data record from an instrument to SSDS * * @param deviceID * is the SSDS Device ID for the instrument that produces the data in payload * @param epochMilliseconds * time of payload sample in milliseconds since 1/1/1970 0000 GMT * @param sequenceNumber * an incrementing number for each packet * @param payload * data from instrument */ public static void publish(long deviceID, long epochMilliseconds, long sequenceNumber, String payload) { // Create a publisher component PublisherComponent pc = new PublisherComponent(); // Create a new SensorDataPacket SSDSDevicePacket packetToSend = new SSDSDevicePacket(deviceID, payload .getBytes().length); // Set the packet type to data (0 = Metadata, 1 = Data, 2 = Message) packetToSend.setPacketType(1); // Assign the time packetToSend.setSystemTime(epochMilliseconds); // Set the parentID to 0 for a parentless deployment packetToSend.setPlatformID(0); // Set the metadataref number to 0, if the data format changes and we can // refer to a different metadata packet then this number will change. packetToSend.setMetadataRef(0); // Set the sequence number packetToSend.setSequenceNo(sequenceNumber); // Set the payload packetToSend.setDataBuffer(payload.getBytes()); // Set the record type to 1 as this is the most common situation packetToSend.setRecordType(1); try { pc.publishBytes(SSDSDevicePacket .convertToPublishableByteArray(packetToSend)); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } /** * Test of SsdsPublisher.publish() * * @param args * No arguments are taken. Test values hard coded. */ public static void main(String[] args) { /* * Example packet for EITS instrument */ long eitsDeviceID = 1696; // 1696 is actual SSDS deviceID for EITS long sampleTime = 1228431283209L; // Sample time: 2008-12-04 22:54:43 long sampleSequenceNumber = 1; // May set it to ID from DB insert // Payload values must match the RecordVariables in SSDS String samplePayload = "12.2, 4.3, 768.2, 2.3, 1.2, 0.2"; // Temp, Cond, Pres, U, V, W SsdsPublisher.publish(eitsDeviceID, sampleTime, sampleSequenceNumber, samplePayload); } } {code} # An example of a shell script (the getM1 OASIS download) calling a perl script ssdsSubmit.pl to publish download statistics records to SSDS:   {code} # Record start of download set starttime_es = `perl -e 'print time, "\n"'` # Download the data and record error status $DOWNLOAD $DOWNLOAD_OPTS >& $DATAFILE set rtnsts = $status # Record the end of download set endtime_es = `perl -e 'print time, "\n"'` # Get the raw data filesize set filesize = `ls -l $DATAFILE | sed -f $BIN/fs.sed` # Log download statistics and submit to SSDS - IDs are unique for mooring deployment set deviceId = 1698 set parentId = 1306 /oasis/bin/ssdsSubmit.pl $deviceId $parentId "$starttime_es,$endtime_es,$filesize,$rtnsts" {code} # If using Java incorporate the above class into a Java application that reads the data from the instrument and calls a method like the main() example for each record # Set up the application to execute the loads for the duration of the deployment # Configure DStoNetCDF.pl cron job execution as is done for the OASIS data processing within the ssdsadmin account on elvis. (For MARS deployments this is done for the DataProducerGroup MARS2008; see the processCurrentMooringDataStreams.sh script in /u/ssdsadmin/dev/DPforSSDS/cimt/). \\]]> 8356161 8388913 {code} # It's recommended that the final XML is to be checked into the puckxml module in MBARI's CVS on moonjelly. # Submit the Metadata to SSDS using the SSDSLoads application (available at [http://new-ssds.mbari.org/ssds-docs/client/]. The \-h option provides a usage note): {code} java -jar ssdsLoads-new-ssds.jar -d 1696.xml {code} ## If a mistake is made in the metadata (e.g. forgetting the platform Deployment) it may be easier to undo the submission and start over - do a deep delete on the parent deployment if this is the case. ## Minor attribute fixes may be done by editing the database once the deployments have been loaded # Check that the metadata has been successfully loaded using the Explorer application. h2. Establish data publishing application # This step assumes that there will be some application reading data from the deployed instrument. Perhaps it is a legacy application which reads the data to load into a custom data storage or visualization system. For this application to publish data to SSDS it must have visibility of each record the instrument produces as that record needs to be "packaged" into a SensorDataPacket and "published" to SSDS. # An example Java application that will package and publish a record is below: {code} import java.io.IOException; import moos.ssds.jms.PublisherComponent; import moos.ssds.transmogrify.SSDSDevicePacket; /** *

* Publish instrument data records to the SSDS database. The client must provide * SSDS device ID, timeStamp, sequence number, and payload. *

*
* * @author : $Author: mccann $ * @version : $Revision: 1.17.2.7 $ *
*

* The * Monterey Bay Aquarium Research Institute (MBARI) provides this * documentation and code "as is", with no warranty, express * or implied, of its quality or consistency. It is provided without * support and without obligation on the part of MBARI to assist in its * use, correction, modification, or enhancement. This information * should not be published or distributed to third parties without * specific written permission from MBARI. *

*
* Copyright 2008 MBARI.
* MBARI Proprietary Information. All rights reserved.
*
*
*/ /** * @author mccann * */ public class SsdsPublisher { /** * Publish data record from an instrument to SSDS * * @param deviceID * is the SSDS Device ID for the instrument that produces the data in payload * @param epochMilliseconds * time of payload sample in milliseconds since 1/1/1970 0000 GMT * @param sequenceNumber * an incrementing number for each packet * @param payload * data from instrument */ public static void publish(long deviceID, long epochMilliseconds, long sequenceNumber, String payload) { // Create a publisher component PublisherComponent pc = new PublisherComponent(); // Create a new SensorDataPacket SSDSDevicePacket packetToSend = new SSDSDevicePacket(deviceID, payload .getBytes().length); // Set the packet type to data (0 = Metadata, 1 = Data, 2 = Message) packetToSend.setPacketType(1); // Assign the time packetToSend.setSystemTime(epochMilliseconds); // Set the parentID to 0 for a parentless deployment packetToSend.setPlatformID(0); // Set the metadataref number to 0, if the data format changes and we can // refer to a different metadata packet then this number will change. packetToSend.setMetadataRef(0); // Set the sequence number packetToSend.setSequenceNo(sequenceNumber); // Set the payload packetToSend.setDataBuffer(payload.getBytes()); // Set the record type to 1 as this is the most common situation packetToSend.setRecordType(1); try { pc.publishBytes(SSDSDevicePacket .convertToPublishableByteArray(packetToSend)); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } /** * Test of SsdsPublisher.publish() * * @param args * No arguments are taken. Test values hard coded. */ public static void main(String[] args) { /* * Example packet for EITS instrument */ long eitsDeviceID = 1696; // 1696 is actual SSDS deviceID for EITS long sampleTime = 1228431283209L; // Sample time: 2008-12-04 22:54:43 long sampleSequenceNumber = 1; // May set it to ID from DB insert // Payload values must match the RecordVariables in SSDS String samplePayload = "12.2, 4.3, 768.2, 2.3, 1.2, 0.2"; // Temp, Cond, Pres, U, V, W SsdsPublisher.publish(eitsDeviceID, sampleTime, sampleSequenceNumber, samplePayload); } } {code} # Incorporate this class into a Java application that reads the data from the instrument and calls a method like the main() example for each record # Set up the application to execute for the duration of the deployment # Configure DStoNetCDF.pl cron job execution as is done for the OASIS data processing within the ssdsadmin account on elvis. (For MARS deployments this is done for the DataProducerGroup MARS2008; see the processCurrentMooringDataStreams.sh script in /u/ssdsadmin/dev/DPforSSDS/cimt/).\\]]> 8356160 15663532 15630775 15663538 15630781 15663536 15630779 8388916 {code} # It's recommended that the final XML is to be checked into the puckxml module in MBARI's CVS on moonjelly. # Submit the Metadata to SSDS using the SSDSLoads application (available at [http://new-ssds.mbari.org/ssds-docs/client/]. The \-h option provides a usage note): {code} java -jar ssdsLoads-new-ssds.jar -d 1696.xml {code} ## If a mistake is made in the metadata (e.g. forgetting the platform Deployment) it may be easier to undo the submission and start over - do a deep delete on the parent deployment if this is the case. ## Minor attribute fixes may be done by editing the database once the deployments have been loaded # Check that the metadata has been successfully loaded using the Explorer application. h2. Establish data publishing application # This step assumes that there will be some application reading data from the deployed instrument. Perhaps it is a legacy application which reads the data to load into a custom data storage or visualization system. For this application to publish data to SSDS it must have visibility of each record the instrument produces as that record needs to be "packaged" into a SensorDataPacket and "published" to SSDS. # An example Java application that will package and publish a record is below: {code} import java.io.IOException; import moos.ssds.jms.PublisherComponent; import moos.ssds.transmogrify.SSDSDevicePacket; /** *

* Publish instrument data records to the SSDS database. The client must provide * SSDS device ID, timeStamp, sequence number, and payload. *

*
* * @author : $Author: mccann $ * @version : $Revision: 1.17.2.7 $ *
*

* The * Monterey Bay Aquarium Research Institute (MBARI) provides this * documentation and code "as is", with no warranty, express * or implied, of its quality or consistency. It is provided without * support and without obligation on the part of MBARI to assist in its * use, correction, modification, or enhancement. This information * should not be published or distributed to third parties without * specific written permission from MBARI. *

*
* Copyright 2008 MBARI.
* MBARI Proprietary Information. All rights reserved.
*
*
*/ /** * @author mccann * */ public class SsdsPublisher { /** * Publish data record from an instrument to SSDS * * @param deviceID * is the SSDS Device ID for the instrument that produces the data in payload * @param epochMilliseconds * time of payload sample in milliseconds since 1/1/1970 0000 GMT * @param sequenceNumber * an incrementing number for each packet * @param payload * data from instrument */ public static void publish(long deviceID, long epochMilliseconds, long sequenceNumber, String payload) { // Create a publisher component PublisherComponent pc = new PublisherComponent(); // Create a new SensorDataPacket SSDSDevicePacket packetToSend = new SSDSDevicePacket(deviceID, payload .getBytes().length); // Set the packet type to data (0 = Metadata, 1 = Data, 2 = Message) packetToSend.setPacketType(1); // Assign the time packetToSend.setSystemTime(epochMilliseconds); // Set the parentID to 0 for a parentless deployment packetToSend.setPlatformID(0); // Set the metadataref number to 0, if the data format changes and we can // refer to a different metadata packet then this number will change. packetToSend.setMetadataRef(0); // Set the sequence number packetToSend.setSequenceNo(sequenceNumber); // Set the payload packetToSend.setDataBuffer(payload.getBytes()); // Set the record type to 1 as this is the most common situation packetToSend.setRecordType(1); try { pc.publishBytes(SSDSDevicePacket .convertToPublishableByteArray(packetToSend)); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } /** * Test of SsdsPublisher.publish() * * @param args * No arguments are taken. Test values hard coded. */ public static void main(String[] args) { /* * Example packet for EITS instrument */ long eitsDeviceID = 1696; // 1696 is actual SSDS deviceID for EITS long sampleTime = 1228431283209L; // Sample time: 2008-12-04 22:54:43 long sampleSequenceNumber = 1; // May set it to ID from DB insert // Payload values must match the RecordVariables in SSDS String samplePayload = "12.2, 4.3, 768.2, 2.3, 1.2, 0.2"; // Temp, Cond, Pres, U, V, W SsdsPublisher.publish(eitsDeviceID, sampleTime, sampleSequenceNumber, samplePayload); } } {code} # An example of a shell script (the getM1 OASIS download) calling a perl script [^ssdsSubmit.pl] to publish download statistics records to SSDS:   {code} # Record start of download set starttime_es = `perl -e 'print time, "\n"'` # Download the data and record error status $DOWNLOAD $DOWNLOAD_OPTS >& $DATAFILE set rtnsts = $status # Record the end of download set endtime_es = `perl -e 'print time, "\n"'` # Get the raw data filesize set filesize = `ls -l $DATAFILE | sed -f $BIN/fs.sed` # Log download statistics and submit to SSDS - IDs are unique for mooring deployment set deviceId = 1698 set parentId = 1306 /oasis/bin/ssdsSubmit.pl $deviceId $parentId "$starttime_es,$endtime_es,$filesize,$rtnsts" {code} # If using Java incorporate the above class into a Java application that reads the data from the instrument and calls a method like the main() example for each record # Set up the application to execute the loads for the duration of the deployment # Configure DStoNetCDF.pl cron job execution as is done for the OASIS data processing within the ssdsadmin account on elvis. (For MARS deployments this is done for the DataProducerGroup MARS2008; see the processCurrentMooringDataStreams.sh script in /u/ssdsadmin/dev/DPforSSDS/cimt/). \\]]> 8356163 15663553 15630796 15663560 15630803 15663562 15630805 15663561 15630804 15663555 15630798 15663558 15630801 15663557 15630800 15663568 15630811 3670735 3637988 15663569 15630812 15663564 15630807 15663566 15630809 3670742 3637995 3670739 3637992 3670754 3638007 3670752 3638005 3670750 3638003 3670759 3638012 8388965 0, the service will consider any time between // samples that is less than the specified gap plus this margin, // it will assume that it is not a gap condition. This is to // prevent false positives when the sample timestamps aren't exactly // on the interval. String gapSpec, // There are two ways to specify a gap: // "serviceCalculated" or "userSpecified" Long numberOfRecords, // If the call specifies SERVICE_CALCULATED and this // is greater than 0, the service will use 'numberOfRecords' // most recent records of the specified RecordType // in calculating the average time between samples Date intervalCalcStartWindow, // This is the date that starts the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this is ignored. Date intervalCalcEndWindow, // This is the date that ends the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this will be used as // the endtime and then the service will use the numberOfRecords // before this time as the data to calculate the average // time interval. Long gapInMillis // If the gapSpec is USER_SPECIFIED, then the service will use // this number of milliseconds as the gap for identifying gaps. ) {code} With a return that has the format of: Properties Objects with properties: ||Property Name||Value|| |lastPacketDateTime|This is the date and time of the last packet received| |totalNumberOfRecords|This is the total number of records for the parameters specified| |numberOfFuturePackets|This is the number of packet that appear in the future. This should be zero and if they are not, there could be bad data| |averagSampleIntervalInMillis|This is the number of milliseconds that the service used to find data gaps| |marginInMillis|This is the number of milliseconds as a margin that the service used to find data gaps| |numRecordsSearchedForGaps|This is the number of records that were searched through while trying to find gaps using the gap criteria| |dataGap1Start|This is the date and time of start of the first possible gap in the data| |dataGap1End|This is the date and time of end of the first possible gap in the data| |.|.| |.|.| |.|.| |dataGapNStart|This is the date and time of start of the Nth possible gap in the data| |dataGapNEnd|This is the date and time of end of the Nth possible gap in the data| h5. Java EJB client h5. REST client If you want to use the REST-style interface the HTTP call would looks something like: {code} http://localhost:8080/servlet/DataAccessServlet?objectToInvokeOn=SQLDataStreamRawDataAccess&method=getDataStreamProperties &p1Type=String&p1Value=1300 &p2Type=Long&p2Value=1 &p3Type=Boolean&p3Value=true &p4Type=Date&p4Value=2008-12-00T00:00:00Z &p5Type=Date&p5Value=2008-01-00T00:00:00Z &p6Type=String&p6Value=timeGap &p7Type=Long&p7Value=5000 &p8Type=String&p8Value=userSpecified &p9Type=Long&p9Value=10 &p10Type=Date&p10Value= &p11Type=Date&p11Value= &p12Type=Long&p12Value=10000 {code} And the return might look something like {code} java.util.Properties{} {code} h5. Web Service Client {panel}]]> 8356217 8388964 0, the service will consider any time between // samples that is less than the specified gap plus this margin, // it will assume that it is not a gap condition. This is to // prevent false positives when the sample timestamps aren't exactly // on the interval. String gapSpec, // There are two ways to specify a gap: // SERVICE_CALCULATED or USER_SPECIFIED Long numberOfRecords, // If the call specifies SERVICE_CALCULATED and this // is greater than 0, the service will use 'numberOfRecords' // most recent records of the specified RecordType // in calculating the average time between samples Date intervalCalcStartWindow, // This is the date that starts the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this is ignored. Date intervalCalcEndWindow, // This is the date that ends the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this will be used as // the endtime and then the service will use the numberOfRecords // before this time as the data to calculate the average // time interval. Long gapInMillis // If the gapSpec is USER_SPECIFIED, then the service will use // this number of milliseconds as the gap for identifying gaps. ) {code} With a return that has the format of: Properties Objects with properties: ||Property Name||Value|| |lastPacketDateTime|This is the date and time of the last packet received| |totalNumberOfRecords|This is the total number of records for the parameters specified| |numberOfFuturePackets|This is the number of packet that appear in the future. This should be zero and if they are not, there could be bad data| |averagSampleIntervalInMillis|This is the number of milliseconds that the service used to find data gaps| |marginInMillis|This is the number of milliseconds as a margin that the service used to find data gaps| |numRecordsSearchedForGaps|This is the number of records that were searched through while trying to find gaps using the gap criteria| |dataGap1Start|This is the date and time of start of the first possible gap in the data| |dataGap1End|This is the date and time of end of the first possible gap in the data| |.|.| |.|.| |.|.| |dataGapNStart|This is the date and time of start of the Nth possible gap in the data| |dataGapNEnd|This is the date and time of end of the Nth possible gap in the data| h5. Java EJB client h5. REST client If you want to use the REST-style interface the HTTP call would looks something like: {code} http://localhost:8080/servlet/DataAccessServlet?objectToInvokeOn=SQLDataStreamRawDataAccess&method=getDataStreamProperties &p1Type=String&p1Value=1300 &p2Type=Long&p2Value=1 &p3Type=String&p3Value=10 &p4Type=Boolean&p4Value=false &p5Type=String&p5Value=0 {code} And the return might look something like {code} java.util.Properties{} {code} h5. Web Service Client {panel}]]> 8356216 3670761 3638014 3670755 3638008 3670757 3638010 3670767 3638020 3670769 3638022 3670763 3638016 3670765 3638018 3670776 3638029 3670784 3638037 3670786 3638039 9405156 tag. For instance any nominalLat/Lon/Depth. {color:#ff0000}The one exception is the nominalDepth, if it is known please set it{color}. # If the tag has a 'name' attribute, make sure it does not have any deployment specific information in it. For example, 'ISUS Deployment' is better than 'M2 ISUS Deployment'. The reason for removing any deployment information from the XML is so that when the device moves to a different mooring, the user should not have to edit the XML. The goal is to get all the XML to a point where it never needs to be edited when an instrument is deployed (unless something in the way the data stream is generated from the instrument changes). # Go to the SSDS Device pages and verify that the all the device information (mfg, model, serial number, name, type, etc.) matches what is currently in SSDS. If any of those are different it will update the device information in SSDS when the XML comes in the data stream. # Verify RecordDescription and RecordVariables look correct. I usually go to the raw data pages in SSDS and bring up the last few packets from the device just to verify that the number of columns and bufferSeparator look about right. # Check any changes to the XML back into CVS. # Copy the XML to the \\Tornado\ssdsdata\mooring(m1\|m2)\YYYY\xml directory # Go to the \\Tornado\ssdsdata\mooring(m1\|m2)\YYYY\cfg directory. # This next step is the one that needs to be timed with the mooring turn. When the old instrument is shutdown: ## Open the ssds.cfg file in a text editor ## Find the line that shows the currently deployed instrument and copy it to a line just below it. For example, if we are replacing the GPS, it might look like this before: {panel:title=Before Copy} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} and this after: {panel:title=After Copy} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # Now change the new line to have the {color:#cc0000}{+}correct device ID{+}{color} *and* the {color:#cc0000}{+}correct XML{+}{color} file URL {panel:title=After Device ID update} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = GPS_TYPE3,1511,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1511.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # To clean up the previous deployment information, put start and end dates after the XML URL {panel:title=After Adding Start/End dates} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,2007/04/25 16:21:58,2007/08/01 10:00:00,TransformGPS instrument = GPS_TYPE3,1511,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1511.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} (Be careful on the format for the start and end date strings.  Leading 0s are required.) # Save the cfg file. {note:title=Saving the file will make the change take hold}When the ssds.cfg file changes (saved) is when the OASIS2SSDS processing will pick up the instrument change. Now, the next time it runs it will pick up the instrument change, grab the XML file from the 'xml' directory and publish it to SSDS. It will then publish all data under the new device ID. {note} # Edit the metadata in SSDS to put a close date on the old instrument deployment in SSDS. Currently I do that using Enterprise Manager. h5. If this is being done after the fact the next steps will also need to be taken. # After the new deployment shows up in SSDS (which can take up to 5 minutes after the OASIS2SSDS has completed), the start time for the new deployment will need to be edited to match the actual time the instrument was swapped. # Also because the data was being published under the incorrect device ID, it will need to be moved from one database table in SSDS_Data on Solstice to another table. ## The first thing that I do is grab the timestamp from the last packet sent from the old device in the raw data page on SSDS. ## For example, I go to: [http://new-ssds.mbari.org:8080/ssds/siamRawDataStep1.jsp] and enter the old device ID and set the number of packets back to make sure it goes far enough back to cover the actual time of the instrument swap. Then click on 'Next->'. ## Once the raw data shows up, find the last packet from the old device and grab the 'SIAM Timestamp' value (not the date/time) as that will be used in the Enterprise Manager query. ## Open Enterprise Manager and navigate to the 'SSDS_Data' database on Solstice. ## Browse the tables and find the table with the device ID of the old device and right click on it and select 'Open Table->Return all rows'. ## Click on the 'SQL' button in Enterprise Manager to bring up the SQL pane. It should show the basic query which should look something like this: {noformat} SELECT * FROM [1416] {noformat} ## Now add the where clause to pick only the data that is after the timestamp you grabbed from the last packet on the web page. {note:title=Timestamps in SQL are in Seconds}A quick note here, the 'SIAM Timestamp' on the raw data page is actually in milliseconds and the database column is in seconds so you will have to remove the last three digits of the 'SIAM Timestamp' before putting it in this query. {note} {noformat} SELECT * FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ## Run this query by clicking the run button '\!' in Enterprise Manager. ## Look over the results to make sure they look about right (usually you are looking for the length of the return which should be much shorter). You can actually use a count query to see how many rows this query will return. A count query would look like: {noformat} SELECT count(*) FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ## Once you know the query is correct (also compare sequence number in query return and the raw data page), copy it to the clipboard and close the query window in Enterprise Manager. ## Navigate to the table of the device you want to copy the data into and right click and select 'All Tasks->Import Data...' which will fire up the DTS wizard. ### Click on 'Next>' ### For the Data Source database choose Solstice ### Select the 'SSDS_Data' database (note you should have permissions to do all this and use your windows authentication) ### Click on 'Next>' ### The destination configuration should be already to go (Solistice and SSDS_Data database). ### Click on 'Next>' ### Select 'Use a query to specify the data to transfer' ### Click on 'Next>' ### Paste the query from your clipboard into the 'Query Statement' window (You can click on 'Parse' if you want a quick sanity check) ### Click on 'Next>' ### Click on the 'Results' entry under the 'Destination' column which will enable a drop down box. ### Choose the table of the newly installed device where you will be copying the data to. ### Click on 'Next>' ### Click on 'Next>' ### Click on 'Finish' which will copy the data. ### Once that is done, open the table of the old instrument and the SQL pane so that we can construct the delete query on the old data. ### Paste in the select query and verify it is the same data you copied over: {noformat} SELECT * FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ### If it looks good, click on the 'Change Query type ...' button in Enterprise Manager and select 'Delete'. This will change the query to a delete query. ### Run the query by click on the run '\!' button. That will remove all the data from the old instrument. That's it ... whew\! Kevin Gomes (August 3, 2007)]]> 9372424 631 633 632 634 634 636 628 630 629 631 630 632 16810016 16777248 16810018 16777250 635 637 636 (is this correct?) * Andrew: 20 days 1. Need project schedule for 2006(this week) 1. Development * General * New machine order HP dual athalon with Red Hat * Need to draft up deployment and migration scenario for new architecture/new machine * Web Application * Sputtering to life ([http://ssdsdevpc:8080/ssds http://ssdsdevpc:8080/ssds]) * Core Metadata * Core Data * Clients * Model Integration 1. MSE * Need to document requirements for data access (by tomorrow!) * No update on data policy 1. ASAP * Need to meet with Mike G to discuss schema changes. == Action Items == 1. Getting new architecture rolled out is critical (time wise) 1. Finish up DTS job migration 1. Get project plan done and identify tasks for developers 1. Look at [https://eventbus.dev.java.net/ eventbus:] as possible solution for remote model object notificaton (and other SSDS stuff). 1. Finish getting John's development environment going 1. Get ASAP update from Mike G. ]]> 638 16810025 16777257 16810023 16777255 16810021 16777253 16810019 16777251 16810027 16777259 11830026 metadataRef # metadataRef->metadataSequenceNumber # metadataRef->dataDescriptionID | # metadataRef # metadataSequenceNumber # dataDescriptionID | | parentId | direct copy: # parentId->parentId # parentId->platformID | # parentId # platformID | | recordType | # MetadataPacket: recordType to 0 # Other: direct copy | recordType | | | # If MetadataPacket, packetType = 0 # If SensorDataPacket, packetType = 1 # If DeviceMessagePacket, packetType = 2 | packetType | | firstBufferLength | ignored | | | firstBuffer | First buffer depends on which type of packet # If MetadataPacket, copy "bytes" buffer # If SensorDataPacket, copy "dataBuffer" # If DeviceMessagePacket, copy "message" | firstBuffer | | secondBufferLength | ignored | | | secondBuffer | Only exists if MetadataPacket and will copy over "cause" buffer | secondBuffer | h6. DevicePacket to SIAM Byte Array This is done by the SIAM Exportable Packet class || DevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataRef | direct copy | metadataRef | | parentId | direct copy | parentId | | recordType | direct copy | recordType | | | This is set based on what type of packet: # If MetadataPacket, set to 0x0101 # If SensorDataPacket, set to 0x0102 # If DeviceMessagePacket, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | firstBufferLength | direct copy (see note for first buffer) | firstBufferLength | | firstBuffer | This depends on the type of packet: # If MetadataPacket, "cause" bytes are copied over # If SensorDataPacket, "dataMessage" bytes are copied over # If DeviceMessagePacket, "message" bytes are copied over | firstBuffer | | secondBufferLength | only valid with MetadataPacket, but is copied directly over | secondBufferLength | | secondBuffer | only valid with MetadataPacket, and the "buffer" bytes are copied over | secondBuffer | h6. SSDSDevicePacket to SIAM Byte Array *Originally done in TransmogrifyMDB by calling SSDSDevicePacket.convertToPublishableVersion3ByteArray before passing byte array to method to translate to SSDS format and then send to ingest* || SSDSDevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataSequenceNumber | direct copy | metadataRef | | parentID | direct copy | parentID | | recordType | # If packetType = 0, set recordType = 0 # If packetType = 1, set recordType = recordType # If packetType = 2, set recordType = recordType | recordType | | | This is set based on what type of packet: # If packetType = 0, set to 0x0101 # If packetType = 1, set to 0x0102 # If packetType = 2, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | | This depends on the type of packet: # If packetType = 0, set to length of "otherBuffer" # If packetType = 1, set to length of "dataBuffer" # If packetType = 2, set to length of "dataBuffer" | firstBufferLength | | other/dataBuffer | This depends on the type of packet: # If packetType = 0, set to bytes from "otherBuffer" # If packetType = 1, set to bytes from "dataBuffer" # If packetType = 2, set to bytes from "dataBuffer" | firstBuffer | | | This only exists if it is packetType = 0 then it is set to the length of the "dataBuffer" | secondBufferLength | | dataBuffer | This only exists if it is packetType = 0 then it is set to the byte from the "dataBuffer" | secondBuffer | h5. Exploration of Upgrade of Ingest/Transmogrify to AMQP In an effort to allow non-Java clients to send data to SSDS in the form of messages and to upgrade the messaging system to a technology that is more scalable and higher performance, an investigation of AMQP implementations was done. # [Qpid Exploration] ]]> 11797282 11830023 metadataRef # metadataRef->metadataSequenceNumber # metadataRef->dataDescriptionID | # metadataRef # metadataSequenceNumber # dataDescriptionID | | parentId | direct copy: # parentId->parentId # parentId->platformID | # parentId # platformID | | recordType | # MetadataPacket: recordType to 0 # Other: direct copy | recordType | | | # If MetadataPacket, packetType = 0 # If SensorDataPacket, packetType = 1 # If DeviceMessagePacket, packetType = 2 | packetType | | firstBufferLength | ignored | | | firstBuffer | First buffer depends on which type of packet # If MetadataPacket, copy "bytes" buffer # If SensorDataPacket, copy "dataBuffer" # If DeviceMessagePacket, copy "message" | firstBuffer | | secondBufferLength | ignored | | | secondBuffer | Only exists if MetadataPacket and will copy over "cause" buffer | secondBuffer | h6. DevicePacket to SIAM Byte Array This is done by the SIAM Exportable Packet class || DevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataRef | direct copy | metadataRef | | parentId | direct copy | parentId | | recordType | direct copy | recordType | | | This is set based on what type of packet: # If MetadataPacket, set to 0x0101 # If SensorDataPacket, set to 0x0102 # If DeviceMessagePacket, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | firstBufferLength | direct copy (see note for first buffer) | firstBufferLength | | firstBuffer | This depends on the type of packet: # If MetadataPacket, "cause" bytes are copied over # If SensorDataPacket, "dataMessage" bytes are copied over # If DeviceMessagePacket, "message" bytes are copied over | firstBuffer | | secondBufferLength | only valid with MetadataPacket, but is copied directly over | secondBufferLength | | secondBuffer | only valid with MetadataPacket, and the "buffer" bytes are copied over | secondBuffer | h5. Exploration of Upgrade of Ingest/Transmogrify to AMQP In an effort to allow non-Java clients to send data to SSDS in the form of messages and to upgrade the messaging system to a technology that is more scalable and higher performance, an investigation of AMQP implementations was done. # [Qpid Exploration] ]]> 11797279 11830024 metadataRef # metadataRef->metadataSequenceNumber # metadataRef->dataDescriptionID | # metadataRef # metadataSequenceNumber # dataDescriptionID | | parentId | direct copy: # parentId->parentId # parentId->platformID | # parentId # platformID | | recordType | # MetadataPacket: recordType to 0 # Other: direct copy | recordType | | | # If MetadataPacket, packetType = 0 # If SensorDataPacket, packetType = 1 # If DeviceMessagePacket, packetType = 2 | packetType | | firstBufferLength | ignored | | | firstBuffer | First buffer depends on which type of packet # If MetadataPacket, copy "bytes" buffer # If SensorDataPacket, copy "dataBuffer" # If DeviceMessagePacket, copy "message" | firstBuffer | | secondBufferLength | ignored | | | secondBuffer | Only exists if MetadataPacket and will copy over "cause" buffer | secondBuffer | h6. DevicePacket to SIAM Byte Array This is done by the SIAM Exportable Packet class || DevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataRef | direct copy | metadataRef | | parentId | direct copy | parentId | | recordType | direct copy | recordType | | | This is set based on what type of packet: # If MetadataPacket, set to 0x0101 # If SensorDataPacket, set to 0x0102 # If DeviceMessagePacket, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | firstBufferLength | direct copy (see note for first buffer) | firstBufferLength | | firstBuffer | This depends on the type of packet: # If MetadataPacket, "cause" bytes are copied over # If SensorDataPacket, "dataMessage" bytes are copied over # If DeviceMessagePacket, "message" bytes are copied over | firstBuffer | | secondBufferLength | only valid with MetadataPacket, but is copied directly over | secondBufferLength | | secondBuffer | only valid with MetadataPacket, and the "buffer" bytes are copied over | secondBuffer | h6. SSDSDevicePacket to SIAM Byte Array *Originally done in TransmogrifyMDB by calling SSDSDevicePacket.convertToPublishableVersion3ByteArray before passing byte array to method to translate to SSDS format and then send to ingest* || SSDSDevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataSequenceNumber | direct copy | metadataRef | | parentID | direct copy | parentID | | recordType | # If packetType = 0, set recordType = 0 # If packetType = 1, set recordType = recordType # If packetType = 2, set recordType = recordType | recordType | | | This is set based on what type of packet: # If packetType = 0, set to 0x0101 # If packetType = 1, set to 0x0102 # If packetType = 2, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | | This depends on the type of packet: # If packetType = 0, set to length of "otherBuffer" # If packetType = 1, set to length of "dataBuffer" # If packetType = 2, set to length of "dataBuffer" | firstBufferLength | | | This depends on the type of packet: # If packetType = 0, set to bytes from "otherBuffer" # If packetType = 1, set to bytes from "dataBuffer" # If packetType = 2, set to bytes from "dataBuffer" | firstBuffer | | | This only exists if it is packetType = 0 then it is set to the length of the "dataBuffer" | secondBufferLength | | | This only exists if it is packetType = 0 then it is set to the byte from the "dataBuffer" | secondBuffer | h5. Exploration of Upgrade of Ingest/Transmogrify to AMQP In an effort to allow non-Java clients to send data to SSDS in the form of messages and to upgrade the messaging system to a technology that is more scalable and higher performance, an investigation of AMQP implementations was done. # [Qpid Exploration] ]]> 11797280 11830021 metadataRef # metadataRef->metadataSequenceNumber # metadataRef->dataDescriptionID | # metadataRef # metadataSequenceNumber # dataDescriptionID | | parentId | direct copy: # parentId->parentId # parentId->platformID | # parentId # platformID | | recordType | # MetadataPacket: recordType to 0 # Other: direct copy | recordType | | | # If MetadataPacket, packetType = 0 # If SensorDataPacket, packetType = 1 # If DeviceMessagePacket, packetType = 2 | packetType | | firstBufferLength | ignored | | | firstBuffer | First buffer depends on which type of packet # If MetadataPacket, copy "bytes" buffer # If SensorDataPacket, copy "dataBuffer" # If DeviceMessagePacket, copy "message" | firstBuffer | | secondBufferLength | ignored | | | secondBuffer | Only exists if MetadataPacket and will copy over "cause" buffer | secondBuffer | h6. DevicePacket to SIAM Byte Array This is done by the SIAM Exportable Packet class || DevicePacket || Translation Rule || SIAM Byte Array || h5. Exploration of Upgrade of Ingest/Transmogrify to AMQP In an effort to allow non-Java clients to send data to SSDS in the form of messages and to upgrade the messaging system to a technology that is more scalable and higher performance, an investigation of AMQP implementations was done. # [Qpid Exploration] ]]> 11797277 11830019 metadataRef # metadataRef->metadataSequenceNumber # metadataRef->dataDescriptionID | # metadataRef # metadataSequenceNumber # dataDescriptionID | | parentId | direct copy: # parentId->parentId # parentId->platformID | # parentId # platformID | | recordType | # MetadataPacket: recordType to 0 # Other: direct copy | recordType | | | # If MetadataPacket, packetType = 0 # If SensorDataPacket, packetType = 1 # If DeviceMessagePacket, packetType = 2 | packetType | | firstBufferLength | ignored | | | firstBuffer | First buffer depends on which type of packet # If MetadataPacket, copy "bytes" buffer # If SensorDataPacket, copy "dataBuffer" # If DeviceMessagePacket, copy "message" | firstBuffer | | secondBufferLength | ignored | | | secondBuffer | Only exists if MetadataPacket and will copy over "cause" buffer | secondBuffer | h5. Exploration of Upgrade of Ingest/Transmogrify to AMQP In an effort to allow non-Java clients to send data to SSDS in the form of messages and to upgrade the messaging system to a technology that is more scalable and higher performance, an investigation of AMQP implementations was done. # [Qpid Exploration] ]]> 11797275 11830033 metadataRef # metadataRef->metadataSequenceNumber # metadataRef->dataDescriptionID | # metadataRef # metadataSequenceNumber # dataDescriptionID | | parentId | direct copy: # parentId->parentId # parentId->platformID | # parentId # platformID | | recordType | # MetadataPacket: recordType to 0 # Other: direct copy | recordType | | | # If MetadataPacket, packetType = 0 # If SensorDataPacket, packetType = 1 # If DeviceMessagePacket, packetType = 2 | packetType | | firstBufferLength | ignored | | | firstBuffer | First buffer depends on which type of packet # If MetadataPacket, copy "bytes" buffer # If SensorDataPacket, copy "dataBuffer" # If DeviceMessagePacket, copy "message" | firstBuffer | | secondBufferLength | ignored | | | secondBuffer | Only exists if MetadataPacket and will copy over "cause" buffer | secondBuffer | h6. DevicePacket to SIAM Byte Array This is done by the SIAM Exportable Packet class || DevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataRef | direct copy | metadataRef | | parentId | direct copy | parentId | | recordType | direct copy | recordType | | | This is set based on what type of packet: # If MetadataPacket, set to 0x0101 # If SensorDataPacket, set to 0x0102 # If DeviceMessagePacket, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | firstBufferLength | direct copy (see note for first buffer) | firstBufferLength | | firstBuffer | This depends on the type of packet: # If MetadataPacket, "cause" bytes are copied over # If SensorDataPacket, "dataMessage" bytes are copied over # If DeviceMessagePacket, "message" bytes are copied over | firstBuffer | | secondBufferLength | only valid with MetadataPacket, but is copied directly over | secondBufferLength | | secondBuffer | only valid with MetadataPacket, and the "buffer" bytes are copied over | secondBuffer | h6. SSDSDevicePacket to SIAM Byte Array *Originally done in TransmogrifyMDB by calling SSDSDevicePacket.convertToPublishableVersion3ByteArray before passing byte array to method to translate to SSDS format and then send to ingest* || SSDSDevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataSequenceNumber | direct copy | metadataRef | | parentID | direct copy | parentID | | recordType | # If packetType = 0, set recordType = 0 # If packetType = 1, set recordType = recordType # If packetType = 2, set recordType = recordType | recordType | | | This is set based on what type of packet: # If packetType = 0, set to 0x0101 # If packetType = 1, set to 0x0102 # If packetType = 2, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | | This depends on the type of packet: # If packetType = 0, set to length of "otherBuffer" # If packetType = 1, set to length of "dataBuffer" # If packetType = 2, set to length of "dataBuffer" | firstBufferLength | | other/dataBuffer | This depends on the type of packet: # If packetType = 0, set to bytes from "otherBuffer" # If packetType = 1, set to bytes from "dataBuffer" # If packetType = 2, set to bytes from "dataBuffer" | firstBuffer | | | This only exists if it is packetType = 0 then it is set to the length of the "dataBuffer" | secondBufferLength | | dataBuffer | This only exists if it is packetType = 0 then it is set to the byte from the "dataBuffer" | secondBuffer | h5. SSDSDevicePacket to SSDS Byte Array *Originally done in SSDSDevicePacket.convertToVersion3ByteArray* || SSDSDevicePacket || Translation Rule || SSDS Byte Array || | sourceID | direct copy | sourceID | | systemTime | split into: # timestampSeconds # timestampNanoseconds | # timestampSeconds # timestampNanoseconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | ignored | | | parentID | ignored | | | recordType | If packetType = 0, set packetSubType to 0, otherwise set to recordType | packetSubType | | packetType | Depends on packetType: # If packetType = 0, set to 1 # If packetType = 1, set to 0 # If packetType = 2, set to 4 | packetType | | metadataSequenceNumber | direct copy | metadataSequenceNumber | | dataDescriptionVersion | direct copy | dataDescriptionVersion | | platformID | direct copy | parentID | | | copy length of dataBuffer | firstBufferLength | | dataBuffer | direct copy | firstBuffer | | | copy length of otherBuffer | secondBufferLength | | otherBuffer | direct copy | secondBuffer | h6. SIAM Byte Array to SSDS Byte Array *Originally done in TransmogrifyMDB in checkAndPublishBytes method* || SIAM Byte Array || Translation Rules || SSDS Byte Array || | EX_DEVICEPACKET | ignored | | | serialVersionUID | ignored | | | sourceID | direct copy | sourceID | | | Depending on EX_XXXXXXXPACKET: # If MetadataPacket, set packetType to 1 # If SensorDataPacket, set packetType to 0 # If DeviceMessagePacket, set packetType to 4 | packetType | | | This was set using the SIAMMetadataTracker that tried to keep track of real version numbers based on XML in payload | metadataSequencNumber | | systemTime | Split into timestampSeconds and timestampNanoseconds | # timestampSeconds # timestampNanoSeconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | direct copy | dataDescriptionVersion | | parentId | direct copy | parentID | | recordType | If MetadataPacket (determined from EX_XXXXXXXPACKET), recordType set to 0, otherwise set to recordType | packetSubType | | EX_XXXXXXXPACKET | ignored in storage, but used in logic | | | serialVersionUID | ignored | | | first/secondBufferLength | If MetadataPacket (determined from EX_XXXXXXXPACKET), firstBufferLength is set to secondBufferLength so we can flip the "cause" and "buffer" bytes because it just made more sense since the cause was rarely populated. Otherwise set to firstBufferLength | firstBufferLength | | first/secondBuffer | If MetadataPacket (determined from EX_XXXXXXXPACKET), firstBuffer is set to secondBuffer so we can flip the "cause" and "buffer" bytes because it just made more sense since the cause was rarely populated. Otherwise set to firstBuffer | firstBuffer | | firstBufferLength | If MetadataPacket (determined from EX_XXXXXXXPACKET), secondBufferLength is set to firstBufferLength to flip "cause" and "buffer" bytes | secondBufferLength | | firstBuffer | If MetadataPacket (determined from EX_XXXXXXXPACKET), secondBuffer is set to firstBuffer to flip "cause" and "buffer" bytes | secondBuffer | h5. Exploration of Upgrade of Ingest/Transmogrify to AMQP In an effort to allow non-Java clients to send data to SSDS in the form of messages and to upgrade the messaging system to a technology that is more scalable and higher performance, an investigation of AMQP implementations was done. # [Qpid Exploration] ]]> 11797289 11830031 metadataRef # metadataRef->metadataSequenceNumber # metadataRef->dataDescriptionID | # metadataRef # metadataSequenceNumber # dataDescriptionID | | parentId | direct copy: # parentId->parentId # parentId->platformID | # parentId # platformID | | recordType | # MetadataPacket: recordType to 0 # Other: direct copy | recordType | | | # If MetadataPacket, packetType = 0 # If SensorDataPacket, packetType = 1 # If DeviceMessagePacket, packetType = 2 | packetType | | firstBufferLength | ignored | | | firstBuffer | First buffer depends on which type of packet # If MetadataPacket, copy "bytes" buffer # If SensorDataPacket, copy "dataBuffer" # If DeviceMessagePacket, copy "message" | firstBuffer | | secondBufferLength | ignored | | | secondBuffer | Only exists if MetadataPacket and will copy over "cause" buffer | secondBuffer | h6. DevicePacket to SIAM Byte Array This is done by the SIAM Exportable Packet class || DevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataRef | direct copy | metadataRef | | parentId | direct copy | parentId | | recordType | direct copy | recordType | | | This is set based on what type of packet: # If MetadataPacket, set to 0x0101 # If SensorDataPacket, set to 0x0102 # If DeviceMessagePacket, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | firstBufferLength | direct copy (see note for first buffer) | firstBufferLength | | firstBuffer | This depends on the type of packet: # If MetadataPacket, "cause" bytes are copied over # If SensorDataPacket, "dataMessage" bytes are copied over # If DeviceMessagePacket, "message" bytes are copied over | firstBuffer | | secondBufferLength | only valid with MetadataPacket, but is copied directly over | secondBufferLength | | secondBuffer | only valid with MetadataPacket, and the "buffer" bytes are copied over | secondBuffer | h6. SSDSDevicePacket to SIAM Byte Array *Originally done in TransmogrifyMDB by calling SSDSDevicePacket.convertToPublishableVersion3ByteArray before passing byte array to method to translate to SSDS format and then send to ingest* || SSDSDevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataSequenceNumber | direct copy | metadataRef | | parentID | direct copy | parentID | | recordType | # If packetType = 0, set recordType = 0 # If packetType = 1, set recordType = recordType # If packetType = 2, set recordType = recordType | recordType | | | This is set based on what type of packet: # If packetType = 0, set to 0x0101 # If packetType = 1, set to 0x0102 # If packetType = 2, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | | This depends on the type of packet: # If packetType = 0, set to length of "otherBuffer" # If packetType = 1, set to length of "dataBuffer" # If packetType = 2, set to length of "dataBuffer" | firstBufferLength | | other/dataBuffer | This depends on the type of packet: # If packetType = 0, set to bytes from "otherBuffer" # If packetType = 1, set to bytes from "dataBuffer" # If packetType = 2, set to bytes from "dataBuffer" | firstBuffer | | | This only exists if it is packetType = 0 then it is set to the length of the "dataBuffer" | secondBufferLength | | dataBuffer | This only exists if it is packetType = 0 then it is set to the byte from the "dataBuffer" | secondBuffer | h5. SSDSDevicePacket to SSDS Byte Array *Originally done in SSDSDevicePacket.convertToVersion3ByteArray* || SSDSDevicePacket || Translation Rule || SSDS Byte Array || | sourceID | direct copy | sourceID | | systemTime | split into: # timestampSeconds # timestampNanoseconds | # timestampSeconds # timestampNanoseconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | ignored | | | parentID | ignored | | | recordType | If packetType = 0, set packetSubType to 0, otherwise set to recordType | packetSubType | | packetType | Depends on packetType: # If packetType = 0, set to 1 # If packetType = 1, set to 0 # If packetType = 2, set to 4 | packetType | | metadataSequenceNumber | direct copy | metadataSequenceNumber | | dataDescriptionVersion | direct copy | dataDescriptionVersion | | platformID | direct copy | parentID | | | copy length of dataBuffer | firstBufferLength | | dataBuffer | direct copy | firstBuffer | | | copy length of otherBuffer | secondBufferLength | | otherBuffer | direct copy | secondBuffer | h6. SIAM Byte Array to SSDS Byte Array * Originally done in TransmogrifyMDB in checkAndPublishBytes method* || SIAM Byte Array || Translation Rules || SSDS Byte Array || | EX_DEVICEPACKET | ignored | | | serialVersionUID | ignored | | | sourceID | direct copy | sourceID | | | Depending on EX_XXXXXXXPACKET: # If MetadataPacket, set packetType to 1 # If SensorDataPacket, set packetType to 0 # If DeviceMessagePacket, set packetType to 4 | packetType | | | This was set using the SIAMMetadataTracker that tried to keep track of real version numbers based on XML in payload | metadataSequencNumber | | systemTime | Split into timestampSeconds and timestampNanoseconds | # timestampSeconds # timestampNanoSeconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | direct copy | dataDescriptionVersion | | parentId | direct copy | parentID | | recordType | If MetadataPacket (determined from EX_XXXXXXXPACKET), recordType set to 0, otherwise set to recordType | packetSubType | | EX_XXXXXXXPACKET | ignored in storage, but used in logic | | | serialVersionUID | ignored | | | first/secondBufferLength | If MetadataPacket (determined from EX_XXXXXXXPACKET), firstBufferLength is set to secondBufferLength so we can flip the "cause" and "buffer" bytes because it just made more sense since the cause was rarely populated. Otherwise set to firstBufferLength | firstBufferLength | | first/secondBuffer | If MetadataPacket (determined from EX_XXXXXXXPACKET), firstBuffer is set to secondBuffer so we can flip the "cause" and "buffer" bytes because it just made more sense since the cause was rarely populated. Otherwise set to firstBuffer | firstBuffer | | firstBufferLength | If MetadataPacket (determined from EX_XXXXXXXPACKET), secondBufferLength is set to firstBufferLength to flip "cause" and "buffer" bytes | secondBufferLength | | firstBuffer | If MetadataPacket (determined from EX_XXXXXXXPACKET), secondBuffer is set to firstBuffer to flip "cause" and "buffer" bytes | secondBuffer | h5. Exploration of Upgrade of Ingest/Transmogrify to AMQP In an effort to allow non-Java clients to send data to SSDS in the form of messages and to upgrade the messaging system to a technology that is more scalable and higher performance, an investigation of AMQP implementations was done. # [Qpid Exploration] ]]> 11797287 5406936 5374189 11830030 metadataRef # metadataRef->metadataSequenceNumber # metadataRef->dataDescriptionID | # metadataRef # metadataSequenceNumber # dataDescriptionID | | parentId | direct copy: # parentId->parentId # parentId->platformID | # parentId # platformID | | recordType | # MetadataPacket: recordType to 0 # Other: direct copy | recordType | | | # If MetadataPacket, packetType = 0 # If SensorDataPacket, packetType = 1 # If DeviceMessagePacket, packetType = 2 | packetType | | firstBufferLength | ignored | | | firstBuffer | First buffer depends on which type of packet # If MetadataPacket, copy "bytes" buffer # If SensorDataPacket, copy "dataBuffer" # If DeviceMessagePacket, copy "message" | firstBuffer | | secondBufferLength | ignored | | | secondBuffer | Only exists if MetadataPacket and will copy over "cause" buffer | secondBuffer | h6. DevicePacket to SIAM Byte Array This is done by the SIAM Exportable Packet class || DevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataRef | direct copy | metadataRef | | parentId | direct copy | parentId | | recordType | direct copy | recordType | | | This is set based on what type of packet: # If MetadataPacket, set to 0x0101 # If SensorDataPacket, set to 0x0102 # If DeviceMessagePacket, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | firstBufferLength | direct copy (see note for first buffer) | firstBufferLength | | firstBuffer | This depends on the type of packet: # If MetadataPacket, "cause" bytes are copied over # If SensorDataPacket, "dataMessage" bytes are copied over # If DeviceMessagePacket, "message" bytes are copied over | firstBuffer | | secondBufferLength | only valid with MetadataPacket, but is copied directly over | secondBufferLength | | secondBuffer | only valid with MetadataPacket, and the "buffer" bytes are copied over | secondBuffer | h6. SSDSDevicePacket to SIAM Byte Array *Originally done in TransmogrifyMDB by calling SSDSDevicePacket.convertToPublishableVersion3ByteArray before passing byte array to method to translate to SSDS format and then send to ingest* || SSDSDevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataSequenceNumber | direct copy | metadataRef | | parentID | direct copy | parentID | | recordType | # If packetType = 0, set recordType = 0 # If packetType = 1, set recordType = recordType # If packetType = 2, set recordType = recordType | recordType | | | This is set based on what type of packet: # If packetType = 0, set to 0x0101 # If packetType = 1, set to 0x0102 # If packetType = 2, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | | This depends on the type of packet: # If packetType = 0, set to length of "otherBuffer" # If packetType = 1, set to length of "dataBuffer" # If packetType = 2, set to length of "dataBuffer" | firstBufferLength | | other/dataBuffer | This depends on the type of packet: # If packetType = 0, set to bytes from "otherBuffer" # If packetType = 1, set to bytes from "dataBuffer" # If packetType = 2, set to bytes from "dataBuffer" | firstBuffer | | | This only exists if it is packetType = 0 then it is set to the length of the "dataBuffer" | secondBufferLength | | dataBuffer | This only exists if it is packetType = 0 then it is set to the byte from the "dataBuffer" | secondBuffer | h5. SSDSDevicePacket to SSDS Byte Array *Originally done in SSDSDevicePacket.convertToVersion3ByteArray* || SSDSDevicePacket || Translation Rule || SSDS Byte Array || | sourceID | direct copy | sourceID | | systemTime | split into: # timestampSeconds # timestampNanoseconds | # timestampSeconds # timestampNanoseconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | ignored | | | parentID | ignored | | | recordType | If packetType = 0, set packetSubType to 0, otherwise set to recordType | packetSubType | | packetType | Depends on packetType: # If packetType = 0, set to 1 # If packetType = 1, set to 0 # If packetType = 2, set to 4 | packetType | | metadataSequenceNumber | direct copy | metadataSequenceNumber | | dataDescriptionVersion | direct copy | dataDescriptionVersion | | platformID | direct copy | parentID | | | copy length of dataBuffer | firstBufferLength | | dataBuffer | direct copy | firstBuffer | | | copy length of otherBuffer | secondBufferLength | | otherBuffer | direct copy | secondBuffer | h6. SIAM Byte Array to SSDS Byte Array * Originally done in TransmogrifyMDB in checkAndPublishBytes method* || SIAM Byte Array || Translation Rules || SSDS Byte Array || | EX_DEVICEPACKET | ignored | | | serialVersionUID | ignored | | | sourceID | direct copy | sourceID | | systemTime | Split into timestampSeconds and timestampNanoseconds | # timestampSeconds # timestampNanoSeconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | direct copy | dataDescriptionVersion | | parentId | direct copy | parentID | | recordType | If MetadataPacket (determined from EX_XXXXXXXPACKET), recordType set to 0, otherwise set to recordType | recordType | | EX_XXXXXXXPACKET | ignored in storage, but used in logic | | | serialVersionUID | ignored | | | first/secondBufferLength | If MetadataPacket (determined from EX_XXXXXXXPACKET), firstBufferLength is set to secondBufferLength so we can flip the "cause" and "buffer" bytes because it just made more sense since the cause was rarely populated. Otherwise set to firstBufferLength | firstBufferLength | | first/secondBuffer | If MetadataPacket (determined from EX_XXXXXXXPACKET), firstBuffer is set to secondBuffer so we can flip the "cause" and "buffer" bytes because it just made more sense since the cause was rarely populated. Otherwise set to firstBuffer | firstBuffer | | firstBufferLength | If MetadataPacket (determined from EX_XXXXXXXPACKET), secondBufferLength is set to firstBufferLength to flip "cause" and "buffer" bytes | secondBufferLength | | firstBuffer | If MetadataPacket (determined from EX_XXXXXXXPACKET), secondBuffer is set to firstBuffer to flip "cause" and "buffer" bytes | secondBuffer | h5. Exploration of Upgrade of Ingest/Transmogrify to AMQP In an effort to allow non-Java clients to send data to SSDS in the form of messages and to upgrade the messaging system to a technology that is more scalable and higher performance, an investigation of AMQP implementations was done. # [Qpid Exploration] ]]> 11797286 5406935 5374188 11830028 metadataRef # metadataRef->metadataSequenceNumber # metadataRef->dataDescriptionID | # metadataRef # metadataSequenceNumber # dataDescriptionID | | parentId | direct copy: # parentId->parentId # parentId->platformID | # parentId # platformID | | recordType | # MetadataPacket: recordType to 0 # Other: direct copy | recordType | | | # If MetadataPacket, packetType = 0 # If SensorDataPacket, packetType = 1 # If DeviceMessagePacket, packetType = 2 | packetType | | firstBufferLength | ignored | | | firstBuffer | First buffer depends on which type of packet # If MetadataPacket, copy "bytes" buffer # If SensorDataPacket, copy "dataBuffer" # If DeviceMessagePacket, copy "message" | firstBuffer | | secondBufferLength | ignored | | | secondBuffer | Only exists if MetadataPacket and will copy over "cause" buffer | secondBuffer | h6. DevicePacket to SIAM Byte Array This is done by the SIAM Exportable Packet class || DevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataRef | direct copy | metadataRef | | parentId | direct copy | parentId | | recordType | direct copy | recordType | | | This is set based on what type of packet: # If MetadataPacket, set to 0x0101 # If SensorDataPacket, set to 0x0102 # If DeviceMessagePacket, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | firstBufferLength | direct copy (see note for first buffer) | firstBufferLength | | firstBuffer | This depends on the type of packet: # If MetadataPacket, "cause" bytes are copied over # If SensorDataPacket, "dataMessage" bytes are copied over # If DeviceMessagePacket, "message" bytes are copied over | firstBuffer | | secondBufferLength | only valid with MetadataPacket, but is copied directly over | secondBufferLength | | secondBuffer | only valid with MetadataPacket, and the "buffer" bytes are copied over | secondBuffer | h6. SSDSDevicePacket to SIAM Byte Array *Originally done in TransmogrifyMDB by calling SSDSDevicePacket.convertToPublishableVersion3ByteArray before passing byte array to method to translate to SSDS format and then send to ingest* || SSDSDevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataSequenceNumber | direct copy | metadataRef | | parentID | direct copy | parentID | | recordType | # If packetType = 0, set recordType = 0 # If packetType = 1, set recordType = recordType # If packetType = 2, set recordType = recordType | recordType | | | This is set based on what type of packet: # If packetType = 0, set to 0x0101 # If packetType = 1, set to 0x0102 # If packetType = 2, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | | This depends on the type of packet: # If packetType = 0, set to length of "otherBuffer" # If packetType = 1, set to length of "dataBuffer" # If packetType = 2, set to length of "dataBuffer" | firstBufferLength | | other/dataBuffer | This depends on the type of packet: # If packetType = 0, set to bytes from "otherBuffer" # If packetType = 1, set to bytes from "dataBuffer" # If packetType = 2, set to bytes from "dataBuffer" | firstBuffer | | | This only exists if it is packetType = 0 then it is set to the length of the "dataBuffer" | secondBufferLength | | dataBuffer | This only exists if it is packetType = 0 then it is set to the byte from the "dataBuffer" | secondBuffer | h5. SSDSDevicePacket to SSDS Byte Array *Originally done in SSDSDevicePacket.convertToVersion3ByteArray* | SSDSDevicePacket | Translation Rule | SSDS Byte Array | | sourceID | direct copy | sourceID | | systemTime | split into: # timestampSeconds # timestampNanoseconds | # timestampSeconds # timestampNanoseconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | ignored | | | parentID | ignored | | | recordType | If packetType = 0, set packetSubType to 0, otherwise set to recordType | packetSubType | | packetType | Depends on packetType: # If packetType = 0, set to 1 # If packetType = 1, set to 0 # If packetType = 2, set to 4 | packetType | | metadataSequenceNumber | direct copy | metadataSequenceNumber | | dataDescriptionVersion | direct copy | dataDescriptionVersion | | platformID | direct copy | parentID | | | copy length of dataBuffer | firstBufferLength | | dataBuffer | direct copy | firstBuffer | | | copy length of otherBuffer | secondBufferLength | | otherBuffer | direct copy | secondBuffer | h5. Exploration of Upgrade of Ingest/Transmogrify to AMQP In an effort to allow non-Java clients to send data to SSDS in the form of messages and to upgrade the messaging system to a technology that is more scalable and higher performance, an investigation of AMQP implementations was done. # [Qpid Exploration] ]]> 11797284 5406926 5374178 11830036 metadataRef # metadataRef->metadataSequenceNumber # metadataRef->dataDescriptionID | # metadataRef # metadataSequenceNumber # dataDescriptionID | | parentId | direct copy: # parentId->parentId # parentId->platformID | # parentId # platformID | | recordType | # MetadataPacket: recordType to 0 # Other: direct copy | recordType | | | # If MetadataPacket, packetType = 0 # If SensorDataPacket, packetType = 1 # If DeviceMessagePacket, packetType = 2 | packetType | | firstBufferLength | ignored | | | firstBuffer | First buffer depends on which type of packet # If MetadataPacket, copy "bytes" buffer # If SensorDataPacket, copy "dataBuffer" # If DeviceMessagePacket, copy "message" | firstBuffer | | secondBufferLength | ignored | | | secondBuffer | Only exists if MetadataPacket and will copy over "cause" buffer | secondBuffer | h6. DevicePacket to SIAM Byte Array This is done by the SIAM Exportable Packet class || DevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataRef | direct copy | metadataRef | | parentId | direct copy | parentId | | recordType | direct copy | recordType | | | This is set based on what type of packet: # If MetadataPacket, set to 0x0101 # If SensorDataPacket, set to 0x0102 # If DeviceMessagePacket, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | firstBufferLength | direct copy (see note for first buffer) | firstBufferLength | | firstBuffer | This depends on the type of packet: # If MetadataPacket, "cause" bytes are copied over # If SensorDataPacket, "dataMessage" bytes are copied over # If DeviceMessagePacket, "message" bytes are copied over | firstBuffer | | secondBufferLength | only valid with MetadataPacket, but is copied directly over | secondBufferLength | | secondBuffer | only valid with MetadataPacket, and the "buffer" bytes are copied over | secondBuffer | h6. SSDSDevicePacket to SIAM Byte Array *Originally done in TransmogrifyMDB by calling SSDSDevicePacket.convertToPublishableVersion3ByteArray before passing byte array to method to translate to SSDS format and then send to ingest* || SSDSDevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataSequenceNumber | direct copy | metadataRef | | parentID | direct copy | parentID | | recordType | # If packetType = 0, set recordType = 0 # If packetType = 1, set recordType = recordType # If packetType = 2, set recordType = recordType | recordType | | | This is set based on what type of packet: # If packetType = 0, set to 0x0101 # If packetType = 1, set to 0x0102 # If packetType = 2, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | | This depends on the type of packet: # If packetType = 0, set to length of "otherBuffer" # If packetType = 1, set to length of "dataBuffer" # If packetType = 2, set to length of "dataBuffer" | firstBufferLength | | other/dataBuffer | This depends on the type of packet: # If packetType = 0, set to bytes from "otherBuffer" # If packetType = 1, set to bytes from "dataBuffer" # If packetType = 2, set to bytes from "dataBuffer" | firstBuffer | | | This only exists if it is packetType = 0 then it is set to the length of the "dataBuffer" | secondBufferLength | | dataBuffer | This only exists if it is packetType = 0 then it is set to the byte from the "dataBuffer" | secondBuffer | h5. SSDSDevicePacket to SSDS Byte Array *Originally done in SSDSDevicePacket.convertToVersion3ByteArray* || SSDSDevicePacket || Translation Rule || SSDS Byte Array || | sourceID | direct copy | sourceID | | systemTime | ignored during the translation directly, but used through getter methods for seconds and nanoseconds | | | timestampSeconds | direct copy (note that this is _sort of_ a direct copy, there are getter methods on SSDSDevicePacket that convert the systemTime to seconds and nanoseconds when called).| | timestampSeconds | | timestampNanoseconds | direct copy (note that this is _sort of_ a direct copy, there are getter methods on SSDSDevicePacket that convert the systemTime to seconds and nanoseconds when called). | timestampNanoseconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | ignored | | | parentID | ignored | | | recordType | If packetType = 0, set packetSubType to 0, otherwise set to recordType | packetSubType | | packetType | Depends on packetType: # If packetType = 0, set to 1 # If packetType = 1, set to 0 # If packetType = 2, set to 4 | packetType | | metadataSequenceNumber | direct copy | metadataSequenceNumber | | dataDescriptionVersion | direct copy | dataDescriptionVersion | | platformID | direct copy | parentID | | | copy length of dataBuffer | firstBufferLength | | dataBuffer | direct copy | firstBuffer | | | copy length of otherBuffer | secondBufferLength | | otherBuffer | direct copy | secondBuffer | h6. SIAM Byte Array to SSDS Byte Array *Originally done in TransmogrifyMDB in checkAndPublishBytes method* || SIAM Byte Array || Translation Rules || SSDS Byte Array || | EX_DEVICEPACKET | ignored | | | serialVersionUID | ignored | | | sourceID | direct copy | sourceID | | | Depending on EX_XXXXXXXPACKET: # If MetadataPacket, set packetType to 1 # If SensorDataPacket, set packetType to 0 # If DeviceMessagePacket, set packetType to 4 | packetType | | | This was set using the SIAMMetadataTracker that tried to keep track of real version numbers based on XML in payload | metadataSequencNumber | | systemTime | Split into timestampSeconds and timestampNanoseconds | # timestampSeconds # timestampNanoSeconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | direct copy | dataDescriptionVersion | | parentId | direct copy | parentID | | recordType | If MetadataPacket (determined from EX_XXXXXXXPACKET), recordType set to 0, otherwise set to recordType | packetSubType | | EX_XXXXXXXPACKET | ignored in storage, but used in logic | | | serialVersionUID | ignored | | | first/secondBufferLength | If MetadataPacket (determined from EX_XXXXXXXPACKET), firstBufferLength is set to secondBufferLength so we can flip the "cause" and "buffer" bytes because it just made more sense since the cause was rarely populated. Otherwise set to firstBufferLength | firstBufferLength | | first/secondBuffer | If MetadataPacket (determined from EX_XXXXXXXPACKET), firstBuffer is set to secondBuffer so we can flip the "cause" and "buffer" bytes because it just made more sense since the cause was rarely populated. Otherwise set to firstBuffer | firstBuffer | | firstBufferLength | If MetadataPacket (determined from EX_XXXXXXXPACKET), secondBufferLength is set to firstBufferLength to flip "cause" and "buffer" bytes | secondBufferLength | | firstBuffer | If MetadataPacket (determined from EX_XXXXXXXPACKET), secondBuffer is set to firstBuffer to flip "cause" and "buffer" bytes | secondBuffer | h5. Exploration of Upgrade of Ingest/Transmogrify to AMQP In an effort to allow non-Java clients to send data to SSDS in the form of messages and to upgrade the messaging system to a technology that is more scalable and higher performance, an investigation of AMQP implementations was done. # [Qpid Exploration] ]]> 11797292 11830035 metadataRef # metadataRef->metadataSequenceNumber # metadataRef->dataDescriptionID | # metadataRef # metadataSequenceNumber # dataDescriptionID | | parentId | direct copy: # parentId->parentId # parentId->platformID | # parentId # platformID | | recordType | # MetadataPacket: recordType to 0 # Other: direct copy | recordType | | | # If MetadataPacket, packetType = 0 # If SensorDataPacket, packetType = 1 # If DeviceMessagePacket, packetType = 2 | packetType | | firstBufferLength | ignored | | | firstBuffer | First buffer depends on which type of packet # If MetadataPacket, copy "bytes" buffer # If SensorDataPacket, copy "dataBuffer" # If DeviceMessagePacket, copy "message" | firstBuffer | | secondBufferLength | ignored | | | secondBuffer | Only exists if MetadataPacket and will copy over "cause" buffer | secondBuffer | h6. DevicePacket to SIAM Byte Array This is done by the SIAM Exportable Packet class || DevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataRef | direct copy | metadataRef | | parentId | direct copy | parentId | | recordType | direct copy | recordType | | | This is set based on what type of packet: # If MetadataPacket, set to 0x0101 # If SensorDataPacket, set to 0x0102 # If DeviceMessagePacket, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | firstBufferLength | direct copy (see note for first buffer) | firstBufferLength | | firstBuffer | This depends on the type of packet: # If MetadataPacket, "cause" bytes are copied over # If SensorDataPacket, "dataMessage" bytes are copied over # If DeviceMessagePacket, "message" bytes are copied over | firstBuffer | | secondBufferLength | only valid with MetadataPacket, but is copied directly over | secondBufferLength | | secondBuffer | only valid with MetadataPacket, and the "buffer" bytes are copied over | secondBuffer | h6. SSDSDevicePacket to SIAM Byte Array *Originally done in TransmogrifyMDB by calling SSDSDevicePacket.convertToPublishableVersion3ByteArray before passing byte array to method to translate to SSDS format and then send to ingest* || SSDSDevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataSequenceNumber | direct copy | metadataRef | | parentID | direct copy | parentID | | recordType | # If packetType = 0, set recordType = 0 # If packetType = 1, set recordType = recordType # If packetType = 2, set recordType = recordType | recordType | | | This is set based on what type of packet: # If packetType = 0, set to 0x0101 # If packetType = 1, set to 0x0102 # If packetType = 2, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | | This depends on the type of packet: # If packetType = 0, set to length of "otherBuffer" # If packetType = 1, set to length of "dataBuffer" # If packetType = 2, set to length of "dataBuffer" | firstBufferLength | | other/dataBuffer | This depends on the type of packet: # If packetType = 0, set to bytes from "otherBuffer" # If packetType = 1, set to bytes from "dataBuffer" # If packetType = 2, set to bytes from "dataBuffer" | firstBuffer | | | This only exists if it is packetType = 0 then it is set to the length of the "dataBuffer" | secondBufferLength | | dataBuffer | This only exists if it is packetType = 0 then it is set to the byte from the "dataBuffer" | secondBuffer | h5. SSDSDevicePacket to SSDS Byte Array *Originally done in SSDSDevicePacket.convertToVersion3ByteArray* || SSDSDevicePacket || Translation Rule || SSDS Byte Array || | sourceID | direct copy | sourceID | | systemTime | ignored | | | timestampSeconds | direct copy | | timestampSeconds | | timestampNanoseconds | direct copy | timestampNanoseconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | ignored | | | parentID | ignored | | | recordType | If packetType = 0, set packetSubType to 0, otherwise set to recordType | packetSubType | | packetType | Depends on packetType: # If packetType = 0, set to 1 # If packetType = 1, set to 0 # If packetType = 2, set to 4 | packetType | | metadataSequenceNumber | direct copy | metadataSequenceNumber | | dataDescriptionVersion | direct copy | dataDescriptionVersion | | platformID | direct copy | parentID | | | copy length of dataBuffer | firstBufferLength | | dataBuffer | direct copy | firstBuffer | | | copy length of otherBuffer | secondBufferLength | | otherBuffer | direct copy | secondBuffer | h6. SIAM Byte Array to SSDS Byte Array *Originally done in TransmogrifyMDB in checkAndPublishBytes method* || SIAM Byte Array || Translation Rules || SSDS Byte Array || | EX_DEVICEPACKET | ignored | | | serialVersionUID | ignored | | | sourceID | direct copy | sourceID | | | Depending on EX_XXXXXXXPACKET: # If MetadataPacket, set packetType to 1 # If SensorDataPacket, set packetType to 0 # If DeviceMessagePacket, set packetType to 4 | packetType | | | This was set using the SIAMMetadataTracker that tried to keep track of real version numbers based on XML in payload | metadataSequencNumber | | systemTime | Split into timestampSeconds and timestampNanoseconds | # timestampSeconds # timestampNanoSeconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | direct copy | dataDescriptionVersion | | parentId | direct copy | parentID | | recordType | If MetadataPacket (determined from EX_XXXXXXXPACKET), recordType set to 0, otherwise set to recordType | packetSubType | | EX_XXXXXXXPACKET | ignored in storage, but used in logic | | | serialVersionUID | ignored | | | first/secondBufferLength | If MetadataPacket (determined from EX_XXXXXXXPACKET), firstBufferLength is set to secondBufferLength so we can flip the "cause" and "buffer" bytes because it just made more sense since the cause was rarely populated. Otherwise set to firstBufferLength | firstBufferLength | | first/secondBuffer | If MetadataPacket (determined from EX_XXXXXXXPACKET), firstBuffer is set to secondBuffer so we can flip the "cause" and "buffer" bytes because it just made more sense since the cause was rarely populated. Otherwise set to firstBuffer | firstBuffer | | firstBufferLength | If MetadataPacket (determined from EX_XXXXXXXPACKET), secondBufferLength is set to firstBufferLength to flip "cause" and "buffer" bytes | secondBufferLength | | firstBuffer | If MetadataPacket (determined from EX_XXXXXXXPACKET), secondBuffer is set to firstBuffer to flip "cause" and "buffer" bytes | secondBuffer | h5. Exploration of Upgrade of Ingest/Transmogrify to AMQP In an effort to allow non-Java clients to send data to SSDS in the form of messages and to upgrade the messaging system to a technology that is more scalable and higher performance, an investigation of AMQP implementations was done. # [Qpid Exploration] ]]> 11797291 8290315 2 9 2008-12-26 21:20:21.240 2008-12-26 21:20:21.240 675 678 676 679 677 680 678 681 4947969 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms *(8 days - 8 KG)* ## (.5 day) Try to change OASIS to make mooring turns less painful (documentation basically) ## (5 days) Build non-JMS mechanism for users to send data/metadata to SSDS. ## (1 day) Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## (1 day) Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Improve Testing *(5 days - 3 KG, 2MM)* ## (.5 day) Verify (unit tests) that the RecordDescription level parse regular expression works ## (.5 day) Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## (.5 day) Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## (.5 day) Write valid unit test for Object and XMLBuilders ## (.5 day) Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## (.5 day) Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. ## (2 days) Improve Wrapper tests # Documentation *(6 days - 3 KG, 3 MM)* ## (.5 day) Put UML diagram of data model on developer section of web app. ## (.5 day) Finish documenting data packet structure on web pages. ## (5 days) Document Explorer, Admin app and HOOVES h3. Tasks that were descoped from 900823 SSDS Hardening Proposal # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). # Load historical OASIS data into SSDS (data and metadata). *This is important but too big for this, separate project* ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). *IMPORTANT EVEN THOUGH DESCOPED* ]]> 4915201 4947970 Fog-SSDS_Metadata # Prepare for opening to community *(3 days - 3 KG)* ## (.5 day) Put Copyright in all SSDS source code and zip up and make externally available. ## (2 days) Setup Source on public repository # Metadata Integrity Checking/Repairing/Enhancing *(33 days - 11 KG, 10 MM, 12 RS)* ## (3 days) Have updateBot check all resources and if they do not have a ResourceType, use the file extension to map it to a MIMEType and create a ResourceType (this will allow queries for specific resources) ## (1 day) Have updateBot crawl all resources and update contentLength if not specified. ## (10 days) Lookup standard variables and standardUnits from UCAR and have UpdateBot mechanism add them to those without (for NetCDF, try to pull StandardVariable and update SSDS). ## (2 days) Implement a mechanism that updates the true end date/time on the DataContainer for data streams to show the latest data for each stream (should also update the number of records). ## (5 days) Look into having SSDS create "README" type files in the same location as certain DataContainers. ### These could/should be in FGDC format(?) ## (12 days) Refactor and reinstate the SQL integrity checks Rich wrote. # Enhance Access Interfaces *(42 days - 20 KG, 22 MM)* ## (2 days) Develop web page to allow administrators to configure plot creation ## (5 days) Finish implementing all DAOs ### Make sure all methods have associated count method ### Make sure all methods have boolean option for return full graph ### Make sure all methods have capability to specify a sort by field ### Verify returned DataContainer collections should be sorted by start date as default ### Verify implemented query for DataContainer by DataContainerGroup ## (.5 day) Verify PC02 plots are working after M0 turnaround ## (.5 day) Add links to CVS XML on device pages ## (1 day) Need a mechanism for users to upload and store "Resources" in SSDS (some secure/backed up network location) and link them to other resource aware objects. ## (.5 day) In Explorer, truncate long deployment names ## (5 days) Implement more queries in Explorer ### Desire in long term for common 'status reporting' system for instruments (something like an ops portal) ### Find all post products from deployment ### Find all resources of certain types (graphics, log files, calibration files, etc.) ### "I want the Air Temperature (StandardVariable.name) from the DataProducer named "M1" from such and such to such and such a time." ### Given a deployment, can I have a query mechanism that can automatically walk up the tree to find the parent DataProducer of type Deployment? ### Given a deployment, can I walk up the parent tree to look for a parent with a certain role? ### "Give me all QC'd data and realtime data up to now". This would need to pull the best QC'd data for some sampled parameter and then integrate it with the realtime, un-QC'd data from streaming data. ## (5 days) Migrate HOOVES to new architecture ## (22 days) Add HOOVES improvements ### Full edit pages for deployment information ### Tree structure for dataset variables that are functions of depth ### SGT-driven contour plots of ZT data (e.g. ADCP, CTD String) ### Faster variable list generation by using DODS rather than netCDF API ### More consistent use of resourceType contentType info (MIME types) ### Top-level data set display for platform level deployment nodes ### Additional queries: #### by standard variable name #### by lat/lon rubber band box via mini maplet gui interface ### Fix Bugs: #### Window sizing on startup #### thread/hash problem with multiple plots #### Numerics not showing for some data sets # Develop admin application to edit all metadata objects and their relationships *(11 days - 11 KG)* ## One function should be able to change the start time on a DataProducer and have an option to update all the child deployment (deep update) to that same start time. ## Build web pages that allow user to send messages to different topics in the ingest component ## Build web page that will allow a user to load data packets into the database from the file storage mechanism. (user selects device and time window). ## Replace instrument monitoring to read open deployments from SSDS and have configuration options. # Improve Data Ingest Mechanisms *(8 days - 8 KG)* ## (.5 day) Try to change OASIS to make mooring turns less painful (documentation basically) ## (5 days) Build non-JMS mechanism for users to send data/metadata to SSDS. ## (1 day) Put true exception handling in SQLIngest to throw messages when the DB can't be reached. ## (1 day) Change PacketSQLOutput/Input to work with any database (not just MS SQL) # Improve Testing *(5 days - 3 KG, 2MM)* ## (.5 day) Verify (unit tests) that the RecordDescription level parse regular expression works ## (.5 day) Make sure XML dates that are coming out of XMLBuilder/ObjectBuilder cycle can be parsed by the XMLDateFormat. ## (.5 day) Check Object Builder/XML builder to make sure it handles URI/URL/UriString/DODSUrl/X/Y/Zoffsets correctly. ## (.5 day) Write valid unit test for Object and XMLBuilders ## (.5 day) Make sure object Schema/ObjectBuilder/XMLBuilder only use one DataContainer/DataProducer for output, input, consumer, tags. ## (.5 day) Write tests for ResourceBLOB->ObjectBuilder for byte array and verify that it is working correctly. ## (2 days) Improve Wrapper tests # Documentation *(6 days - 3 KG, 3 MM)* ## (.5 day) Put UML diagram of data model on developer section of web app. ## (.5 day) Finish documenting data packet structure on web pages. ## (5 days) Document Explorer, Admin app and HOOVES h3. Tasks that were descoped from 900823 SSDS Hardening Proposal # Setup SPLUNK to monitor SSDS system logs (JBoss, Apache, UNIX system) # Can we setup SSDS so that if a certain process/data container were deemed bad, that you could prevent any reprocessing of that data using that process from happening? # Mike M had new idea about extending our model classes to implement remote lazy loading ... interesting idea and I would like to add that functionality later. # Can I embed the business logic documentation as JavaDoc tags in the source code and then export that somewhere? This is specifically related to the EJB services. # Add end of line terminator as separator in parsing packet records (not files) # Implement fixed format (length) parsing # Implement way to handle multi-record data # Allow user's to "annotate" DataContainer by storing comments # Design architecture for plug-in type functionality for servlet type access (i.e. inline filters for data like calculate salinity on the fly) ## Add a standard capability to SSDS to do things like mx+b, quadratics, using multiple variables in calculations (these should be able to be defined in the XML and SSDS do them automatically) # Refactor navigation menus and scheme for SSDS site and post processed pages (left menu) ## Need to add rest of SSDS web page content to main SSDS pages # Web pages to help with automated workflows(?) # Create web page to do variable/variable plots (with some query capability) # Create page to merge data from different sources (could we use Francisco's help/resources:By Platform,By Instrument,By Variable,By Lat/Lon Box,map with AUV track and moorings/instruments,data from AUV/Moorings/Ships merged by time (plots and raw data) # For application that show lots of graphics, implement thumbnails (click to enlarge) # SIAMRawDataAccess Page -> The list of instruments and parents doesn't show what is current. Maybe I can put a time/date of last update next to the instrument. # Build web form to help developers build URLs that they can use in their code to access data from SSDS # Somehow need to tell SSDS user's that they need to specify an acknowledgement of MBARI when users utilize the data from SSDS. # Implement mechanism (web page, email notification) that allows valid users to approve or map additional relationship and then notify the user of that change so they can change their source. This should be tied into UpdateBot so that it knows what associations it can make between RecordVariable and StandardVarible, for example. ## StandardVariables ## StandardUnits ## StandardKeywords ## StandardDomain ## StandardReferenceScale ## DeviceType ## ResourceType ## DataProducerGroup ## DataContainerGroup # GoogleMaps/GoogleEarth/Worldwind integration # Instead of using command line argument for timezone->UTC, look at [http://www.hibernate.org/100.html Hibernate code] for UTC dates. # Put statistics on plots (min/max of past 24 hrs, latest read value), use max, mins to drive axis range (no autoranging) # Where data are dense enough to make for easy viewing, use scatter instead of line plots, leaving gaps for missing data (if lines must be used, put breaks where data gaps are) # Setup DeviceQCPlotCreator to create plots with multiple lines on one chart (and separate axes). # Make any direction plot (wind, heading, etc.) plot as points, not lines # Put nominal lattitude and longitude in plot titles # Have capability to turn on/off autoscale on plots and specify range # Create servlet that dynamically generates schema and puts in tokens from class constants and database entries. For example, DeviceTypes, ResourceTypes, StandardXXXXXs # Build application to allow users to add QC flags and comments to data packets in SSDS_Data # Convert ingest to pull metadata revision number from XML instead of metadata sequence number from packet # Add mechanism to notify users of new data arrival (method TBD) ## Look into NRSS (RSS for data streams) # In PacketOutputManager, implement a cleanup watchdog (You could do this by keeping another hashmap that had the key deviceID_metadataRevision_recordType_platformID and then a value of last time accessed. Have the watchdog loop through those and PacketOutputs that have not been accessed in some interval get closed and removed from the packetOutputs hashmap) # Add software to crawl SQL packets, read in corresponding GPS data and attach to packet ## (5 days) Remove Deployment info from PUCK XML and move all to new schema and validate (due to the amount of work to do this, it will be done on an as needed basis) # Could we move applications on SSDSPub to another machine with Tomcat and CNAME ssdspub to that machine? # Follow up on PUCK configuration tool (ACE) # *Build services to read data from DataContainers that are files through the query interface (not just from packets).* (This is really valuable, but not REALLY needed) # Look into implementing paging in services (Hibernate supports this). # Load historical OASIS data into SSDS (data and metadata). *This is important but too big for this, separate project* ## Figure out how we will integrate data files offloaded from instruments when they are recovered. (generalize to any data file and this might be a location where the file can be place with appropriate metadata and it will be loaded into SSDS. SSDS can even do some thinking like with NetCDF files). *IMPORTANT EVEN THOUGH DESCOPED* ]]> 4915202 18645288 18579773 18645287 ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 30-May-11 21:59 yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" yes? SET REGION/T="30-May-2011 04:43":"31-May-2011 04:43" yes? list sea_water_temperature_hr VARIABLE : Sea Water Temperature (Celsius) DATA SET : Hourly Gridded MBARI Mooring M1 Sea Water Temperature and Salinity Observations FILENAME : OS_M1_20101027hourly_CMSTV.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 11 by 24 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 30-MAY-2011 04:30 / 5145: 10.77 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 05:30 / 5146: 10.69 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 06:30 / 5147: 10.59 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 07:30 / 5148: 10.49 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 08:30 / 5149: 10.42 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 09:30 / 5150: 10.29 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 10:30 / 5151: 10.23 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 11:30 / 5152: 10.33 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 12:30 / 5153: 10.35 .... .... .... .... .... .... .... .... .... .... 30-MAY-2011 13:30 / 5154: 10.34 .... .... 9.56 9.08 8.85 8.39 8.10 7.89 7.59 7.51 30-MAY-2011 14:30 / 5155: 10.36 .... .... 9.28 8.92 8.70 8.46 8.12 7.90 7.60 7.51 30-MAY-2011 15:30 / 5156: 10.38 .... .... 9.26 8.92 8.73 8.39 8.11 7.93 7.67 7.50 30-MAY-2011 16:30 / 5157: 10.45 .... .... 9.38 9.04 8.81 8.38 8.11 7.93 7.68 7.49 30-MAY-2011 17:30 / 5158: 10.50 .... .... 9.55 9.13 8.89 8.64 8.15 7.96 7.68 7.49 30-MAY-2011 18:30 / 5159: 10.76 .... .... 10.16 9.14 8.90 8.71 8.26 7.97 7.67 7.49 30-MAY-2011 19:30 / 5160: 11.06 .... .... 10.15 9.10 8.91 8.69 8.25 7.98 7.69 7.50 30-MAY-2011 20:30 / 5161: 11.33 .... .... 10.18 9.07 8.90 8.61 8.21 8.01 7.69 7.49 30-MAY-2011 21:30 / 5162: 11.37 .... .... 9.75 9.03 8.83 8.50 8.24 7.99 7.69 7.49 30-MAY-2011 22:30 / 5163: 11.10 .... .... 9.85 9.04 8.83 8.51 8.27 7.99 7.69 7.50 30-MAY-2011 23:30 / 5164: 11.05 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 00:30 / 5165: 10.90 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 01:30 / 5166: 10.84 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 02:30 / 5167: 10.59 .... .... .... .... .... .... .... .... .... .... 31-MAY-2011 03:30 / 5168: 10.37 .... .... .... .... .... .... .... .... .... ....   {noformat} The many '....'s indicate missing data in this hourly gridded file. Let's look upstream in the data processing to see what the input data looks like. The .jnl file that created the 201010/OS_M1_20101027hourly_CMSTV.nc file simply USEd the TS data. Listing the data from the TS file shows the same gappy data as above. So let's look at the input data to the TS file. The jnl file link in the Data column in the _\[TS: \]_ row. Listing the data from the individual input files shows that they apparently do not have the gaps that are shown int he gridded data set: {noformat} yes? USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0010_20101027_original.nc" yes? list temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 140 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 30-MAY-2011 04:49:31 / 30019: 10.73 30-MAY-2011 04:59:30 / 30020: 10.72 30-MAY-2011 05:09:31 / 30021: 10.70 30-MAY-2011 05:19:29 / 30022: 10.70 30-MAY-2011 05:29:30 / 30023: 10.69 30-MAY-2011 05:39:31 / 30024: 10.69 (records skipped) 31-MAY-2011 02:59:30 / 30152: 10.12 31-MAY-2011 03:09:31 / 30153: 10.06 31-MAY-2011 03:19:30 / 30154: 10.08 31-MAY-2011 03:29:30 / 30155: 10.07 31-MAY-2011 03:39:29 / 30156: 10.06 31-MAY-2011 03:49:31 / 30157: 10.08 31-MAY-2011 03:59:30 / 30158: 10.05 {noformat}\\ h4. B. Tips for debugging with Ferret To figure out what is going wrong we'll need to execute more of the Ferret commands from the jnl file that creates the TS file and examine the data at each step. This is an interactive process that involves editing a temporary .jnl file, executing it in ferret with a "GO " and analyzing the output. Here are some more tips for diagnosing problems: # Examine the production ferret output - this is the 'out' link on the current_qcPlots.html web page # Examine CVS for changes in the source code - the change log for combineTS.pl (the script that produces the jnl file link in the Data column in the _\[TS: \]_ row) is [http://moonjelly.shore.mbari.org/cgi-bin/viewvc.cgi/DPforSSDS/cimt/combineTS.pl?view=log] # Examine the data and plots for the individual microcats on the current_qcPlots.html web page # Read the excellent online Ferret documentation, starting with the "Thinking like a Ferret" page: [http://ferret.pmel.noaa.gov/Ferret/documentation/users-guide/introduction/GETTING-STARTED] To follow through with this example of gappy data I took these steps: # Copied the OS_MBARI-M1_20101027_R_TS.jnl file from it's production location (/mbari/ssdsdata/deployments/m1/201010) to /tmp # Edited the file to process and save only the 1m and 10m data and save the date to a temporary netcdf file in /tmp # Executed the temporary truncated file from a Ferret session {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" ! Description: Produce netCDF file of all Temperature and Salinity measurements from a mooring ! Pull data from original instrument netCDF files and grid onto a common grid. ! Automatically generated by ./combineTS.pl on Tue May 31 09:37:27 2011. ! For information on Ferret see http://ferret.wrc.noaa.gov/Ferret/. ! SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:`PSAL,return=lend`/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC !-> LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/KLIMITS=1:11/LLIMITS=1:5182/Z=1/T="27-Oct-2010 21:00:00":"31-May-2011 16:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc **TMAP ERR: error in line definition disordered output coordinate value: 22215. Axis: TIME LIST/FORMAT=CDF/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 20:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC Command file, command group, or REPEAT execution aborted yes? {noformat} This looks like a good clue. The 10m data are not being written to the output file and we are getting this obscure "disordered output coordinate value" error from Ferret. We need to get to the bottom of this. Here are some more tips on how to proceed: # Use the Ferret mail list archive () or Google to search for the meaning of this error message # Join the Ferret mail list and post your question. Solutions are typically provided within a day. # Examine the input data and variables using ncdump(1) or Ferret LIST and SHOW commands Here are some Ferret commands to examine the first 3 temperature values from the 1m and 10m input CTD data and the first 3 times of the output axis: {noformat} yes? list/l=1:3/d=1 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1338 at original sampling intervals FILENAME : m1_ctd0001_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 1 122W 1 27-OCT-2010 21:08:43 / 1: 14.01 27-OCT-2010 21:22:27 / 2: 14.10 27-OCT-2010 21:28:23 / 3: 14.10 yes? list/l=1:3/d=2 temperature VARIABLE : Water Temperature (deg C) DATA SET : Mooring M1 CTD data from MBARI instrument id 1491 at original sampling intervals FILENAME : m1_ctd0010_20101027_original.nc FILEPATH : http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/ SUBSET : 3 points (TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N DEPTH (m): 10 122W 1 27-OCT-2010 20:11:17 / 1: 13.38 27-OCT-2010 20:21:16 / 2: 13.59 27-OCT-2010 20:31:16 / 3: 13.45 yes? show/l=1:3 axis time name axis # pts start end TIME TIME 5181 r 27-OCT-2010 20:30 31-MAY-2011 16:30 T0 = 01-JAN-1950 00:00:00 Axis span (to cell edges) = 215.875 L T TBOX TBOXLO TSTEP (DAYS) 1> 27-OCT-2010 20:30:00 0.0416667 27-OCT-2010 20:00:00 22214.85 2> 27-OCT-2010 21:30:00 0.0416667 27-OCT-2010 21:00:00 22214.9 3> 27-OCT-2010 22:30:00 0.0416667 27-OCT-2010 22:00:00 22214.94 {noformat}\\ h4. C. Fixing the start times for all the child instrument deployments The problem seems to be that the 10m data begin at 20:11:17 before the 1m data at 21:08:43. The first write of the 1m data to the output netCDF file gives the file its shape with the KLIMITS= and LLIMITS= options. Writing the 10m data, which begins in before the bounds that were defined violates Ferret's axis ordering logic. To test this hypothesis edit the SAVE statement in the temporary .jnl file to make the start time for the 10m data 21:00, execute it and then examine the output file: {noformat} yes? go "/tmp/OS_MBARI-M1_20101027_R_TS.jnl" SAVE/FILE="/tmp/OS_MBARI-M1_20101027_R_TS.nc"/APPEND/Z=10/T="27-Oct-2010 21:00:00":"31-May-2011 15:00:00" PSAL, PSAL_QC, TEMP, TEMP_QC, TIME_QC, POSITION_QC, DEPTH_QC LISTing to file /tmp/OS_MBARI-M1_20101027_R_TS.nc (Another ferret session) > ferret NOAA/PMEL TMAP FERRET v6.62 Linux rh5 (gfortran) 2.6.18-164.11.1.el5 - 06/11/10 31-May-11 12:19 yes? use "/tmp/OS_MBARI-M1_20101027_R_TS.nc" yes? list/l=1:10 temp VARIABLE : Hourly sea_water_temperature (celsius) FILENAME : OS_MBARI-M1_20101027_R_TS.nc FILEPATH : /tmp/ SUBSET : 11 by 10 points (DEPTH (m)-TIME) LONGITUDE: 122W(-122) LATITUDE : 36.8N 1 10 20 40 60 80 100 150 200 250 300 1 2 3 4 5 6 7 8 9 10 11 27-OCT-2010 21:30 / 1: 14.09 13.60 .... .... .... .... .... .... .... .... .... 27-OCT-2010 22:30 / 2: 14.17 13.62 .... .... .... .... .... .... .... .... .... 27-OCT-2010 23:30 / 3: 14.34 13.63 .... .... .... .... .... .... .... .... .... 28-OCT-2010 00:30 / 4: 14.28 13.69 .... .... .... .... .... .... .... .... .... 28-OCT-2010 01:30 / 5: 14.22 13.72 .... .... .... .... .... .... .... .... .... 28-OCT-2010 02:30 / 6: 14.21 13.78 .... .... .... .... .... .... .... .... .... 28-OCT-2010 03:30 / 7: 14.21 13.76 .... .... .... .... .... .... .... .... .... 28-OCT-2010 04:30 / 8: 14.09 13.79 .... .... .... .... .... .... .... .... .... 28-OCT-2010 05:30 / 9: 14.01 13.77 .... .... .... .... .... .... .... .... .... 28-OCT-2010 06:30 / 10: 13.89 13.76 .... .... .... .... .... .... .... .... .... yes? {noformat} O.K.\! That looks good. The 10m data got written. Now to fix it for production. This deployment was a special case where the 1m data start after the 10m data. Typically, it's most convenient to have all of the microcats have the same deployment start time. This makes the follow on data processing much simpler. All of the deployment start times in SSDS_METADTA for the M1 201010 deployment were adjusted to be the same as the start time for the Mooring deployment. Performing this step is now part of the [standard operating procedure for performing mooring turns|SSDS:OASIS Mooring turn]. Once SSDS_METADATA has been updated the individual instrument netCDF files must be created by DStoNetCDF.pl before combineTS.pl in run. h4. D. Examining the gridding method There are still some problems with gappy data at 40m and below in the \_TS file. To debug these we'll make another copy of the OS_MBARI-M1_20101027_R_TS.jnl to /tmp and edit it to go through the processing of the 40m data saving the data to a netCDF file in /tmp so as not to disturb the production processing. We also comment out all of the "CANCEL DATA 1" statements so that we can examine data from the files. This file is executed and we see no errors. We need to examine the input data and the Ferret variables that are constructed for the gridding. Here are the commands from the .jnl that perform the the gridding for the 40m data: {noformat} ! ! Remove temperature outliers before gridding ! LET Temperature_QFLAG = IF Temperature GT 2 AND Temperature LT 20 THEN 1 ELSE (-99999) SET VAR/BAD=-99999 Temperature_QFLAG LET Temperature_QC = Temperature_QFLAG * Temperature ! ! Compute my own mean of the data, making sure to assign missing values in the gaps in the Gap FLAG ! Allow at least 1 data point in each destination cell - there are usually 6 for 10 minute data ! LET Temperature_GOOD = Temperature_QC[gt=TIME@SUM] / Temperature_QC[gt=TIME@NGD] LET Temperature_GFLAG = IF Temperature[gt=TIME@NGD,gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] LT 1 THEN (-99999) ELSE 1 SET VAR/BAD=-99999 Temperature_GFLAG LET TEMP = Temperature_GFLAG * Temperature_GOOD[gz=DEPTH@XACT,gy=LATITUDE,gx=LONGITUDE] {noformat} Here are some sample Ferret commands for plotting these data from the session that just executed the OS_MBARI-M1_20101027_R_TS.jnl script: {noformat} yes? go "OS_MBARI-M1_20101027_R_TS.jnl" yes? set region/t=1-may-2011:1-jun-2011 yes? plot Temperature_QC yes? plot/ov/symbol=1 temperature yes? set win 2 yes? plot Temperature_QC[gt=TIME@NGD] yes? plot TEMPERATURE_QC[GT=TIME@SUM] {noformat} These last two plot commands indicate the source of the problem of the gappy data from the inductive modem microcats. Another big clue is the comment in the .jnl file referring to the 10-minute input data and having at least 6 good values within each gridding cell. As can be seen in the plot we often have less than 1 for the @NGD value for the 40m data. These data are hourly and we should be using a different gridding algorithm for them. !m1_ngd_sum.gif|thumbnail! The October 2010 M1 deployment was the first one where all the inductive modem connected CTDs were logged as separate instruments, and the first deployment where the serial microcat processing software was used to process the inductive modem microcat data. The fix for this problem appears to be to do a different gridding for the hourly data from 40m and below. This change was implemented in the code on 2 June 2011 using the Ferret same gridding transform (@MAX) that was previously used for the inductive modem data processing.  Here is the after figure: !wind_SEA_WATER_TEMPERATURE_HR_last30_after.gif|thumbnail! The subsurface data looks a lot better, but there are still some gaps.  If the gaps extend across the whole water column encompassing both the inductive modem and serial microcats then perhaps the gap is due to an interruption in data processing through oasisToSSDS.  This can be remedied by reprocessing the telemetered data by following the README instructions in /u/ssdsadmin/dev/DPforSSDS/oasis/scripts on elvis. h4. E. Testing the results of the gridding It is worthwhile to do another check on the quality of the gridding that is now being performed.  The following Ferret exercice will show the input data compared to the gridded data. {noformat} USE "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/OS_M1_20101027hourly_CMSTV.nc" SET REGION/T="29-May-2011 18:13":"02-Jun-2011 18:13" shade SEA_WATER_TEMPERATURE_HR set win 2 plot/k=1/vlimits=7:13 SEA_WATER_TEMPERATURE_HR repeat/k=2:11 plot/ov SEA_WATER_TEMPERATURE_HR {noformat} Produces these images showing missing data, especially from the 20m and 40m microcats: !m1_shade.gif|thumbnail! !m1_lines.gif|thumbnail! Let's compare the original instruments date from 20m and 40m to the gridded product, using the same Ferret session from above:" {noformat} use "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0020_20101027_original.nc" plot/symbol=1 temperature[d=2] plot/ov SEA_WATER_TEMPERATURE_HR[k=3,d=1] use "http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/201010/m1_ctd0040_20101027_original.nc" plot/symbol=1 temperature[d=3] plot/ov SEA_WATER_TEMPERATURE_HR[k=4,d=1] {noformat} Gives: !m1_compare_20.gif|thumbnail! !m1_compare_40.gif|thumbnail! We obviously still have some gridding issues... h4. F. Rinse and repeat Now is the time to iterate using our method of starting with the .jnl file that aggregates and grids the data to the OceanSITES \_TS.nc file going back to step D. \\]]> 18579772 670 673 674 677 673 676 672 675 11829828 11797075 17891618 17858875 11829830 ALOHA interactions. h5. Timeline: More details to come from ALOHA group, but some timeline notes: # Ship time is January # They want test system in lab ASAP (they actually have some of it running currently) h5. Action Items: # Kevin will look back and try to get SSDS install going again on kainani. # ALOHA group will send Bob information about instruments (IP addresses, etc.) # Bob will look into getting SIAM running against their instruments. # ALOHA group will work on generating a timeline with milestones like code freeze, integration and test, etc. NOTE: As long as the time is not huge, we can charge to TTOS.]]> 11797077 11829831 11797078 17891622 17858879 8945689 0, the service will consider any time between // samples that is less than the specified gap plus this margin, // it will assume that it is not a gap condition. This is to // prevent false positives when the sample timestamps aren't exactly // on the interval. String gapSpec, // There are two ways to specify a gap: // "serviceCalculated" or "userSpecified" Long numberOfRecords, // If the call specifies SERVICE_CALCULATED and this // is greater than 0, the service will use 'numberOfRecords' // most recent records of the specified RecordType // in calculating the average time between samples Date intervalCalcStartWindow, // This is the date that starts the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this is ignored. Date intervalCalcEndWindow, // This is the date that ends the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this will be used as // the endtime and then the service will use the numberOfRecords // before this time as the data to calculate the average // time interval. Long gapInMillis // If the gapSpec is USER_SPECIFIED, then the service will use // this number of milliseconds as the gap for identifying gaps. ) {code} With a return that has the format of: Properties Objects with properties: ||Property Name||Value|| |lastPacketDateTime|This is the date and time of the last packet received| |totalNumberOfRecords|This is the total number of records for the parameters specified| |numberOfFuturePackets|This is the number of packet that appear in the future. This should be zero and if they are not, there could be bad data| |averagSampleIntervalInMillis|This is the number of milliseconds that the service used to find data gaps| |marginInMillis|This is the number of milliseconds as a margin that the service used to find data gaps| |numRecordsSearchedForGaps|This is the number of records that were searched through while trying to find gaps using the gap criteria| |dataGap1Start|This is the date and time of start of the first possible gap in the data| |dataGap1End|This is the date and time of end of the first possible gap in the data| |.|.| |.|.| |.|.| |dataGapNStart|This is the date and time of start of the Nth possible gap in the data| |dataGapNEnd|This is the date and time of end of the Nth possible gap in the data| h5. Java EJB client h5. REST client If you want to use the REST-style interface the HTTP call would looks something like: {code} http://localhost:8080/servlet/DataAccessServlet?objectToInvokeOn=SQLDataStreamRawDataAccess&method=getDataStreamProperties &p1Type=Long&p1Value=1300 &p2Type=Long&p2Value=1 &p3Type=Boolean&p3Value=true &p4Type=Date&p4Value=2008-12-00T00:00:00Z &p5Type=Date&p5Value=2008-01-00T00:00:00Z &p6Type=String&p6Value=timeGap &p7Type=Long&p7Value=5000 &p8Type=String&p8Value=userSpecified &p9Type=Long&p9Value=10 &p10Type=Date&p10Value= &p11Type=Date&p11Value= &p12Type=Long&p12Value=10000 {code} And the return might look something like {code} java.util.Properties{} {code} h5. Web Service Client {panel}]]> 8912922 841 844 837 840 848 851 847 850 844 847 8945692 DataProducer that must be of type "deployment" and then creates "deep" copy of it and persists the new copy. It pulls in the options specified in the incoming parameters to create a unique copy of the DataProducers and DataContainer (outputs). All the rest of the object should be linked to the ones that already exist in the peristent storage. The new ID of the duplicate deployment is returned. h5. The Interface ||Return||Parameters|| |The ID of the newly generated deployment|* DataProducer to Copy * Date the copy should start at * A boolean to indicate if the old (original) should be closed * Date the old (original) should use as an end date (if to be closed) * String that will be the name of the new deployment (copy) * String that is the base of the data stream URI which should be {code}http://new-ssds.mbari.org/servlet/GetOriginalDataServlet{code}| So the API interface looks something like: {code:title=createDuplicateDeepDeployment} public Long createDuplicateDeepDeployment( DataProducer deploymentToCopy, // DataProducer to copy Date newStartDate, // Date to start copied DataProducer on boolean closeOld, // Whether or not old (original) should be closed Date oldEndDate, // Date to use for end date if original is closed String newHeadDeploymentName, // Name of the new DataProducer (copy) String baseDataStreamUri // Base URI for raw data access (http://new-ssds.mbari.org/servlet/GetOriginalDataServlet) ) {code} h5. Java EJB client Under construction h5. REST Style client To use this service via HTTP (REST Style), the call would be constructed using this format: {code:title=REST Style format} http://new-ssds.mbari.org/servlet/MetadataAccessServlet?objectToInvokeOn=DataProducerAccess&method=createDuplicateDeepDeployment &p1Type=DataProducer&p1Value=DataProducer|id=1938292 &p2Type=Date&p2Value=2008-12-01T00:00:00Z &p3Type=boolean&p3Value=true &p4Type=Date&p4Value=2008-11-30T00:00:00Z &p5Type=String&p5Value=New copy of Deployment &p6Type=String&p6Value=http://new-ssds.mbari.org/servlet/GetOriginalDataServlet {code} And the return might looks something like this: {code} java.lang.Long=4552323 {code} h5. Web Services client Under construction {panel} {panel:title=getDataStreamProperties} h5. Background The desire it to have a service that can characterize a DataStream from an instrument. This would allow for easier monitoring of instruments on the network. h5. The Interface The things that the user would want to know are: ||Return||Parameters|| |Date and time of last packet received|* Device ID * RecordType to search for (nothing/default means most recent packet)| |Total Number Of Records|* Device ID * RecordType to search for (nothing/default specified means all packets)| |Data Gaps|* Device ID * RecordType * Time window over data to search for gaps * Gap criteria ** Type of gap *** Time only *** Sequence number only *** Time and sequence number ** Ways to specify gap *** Margin on gap in milliseconds (anything longer than gap + margin will be considered a possible gap) *** Let service calculate gap constraints (calculate average time between sample) **** Number of points to use (points back from most recent packet) **** Or time window to use (start to end time) *** Specify gap constraints **** Gap in milliseconds| So the API interface looks like: {code:title=getDataStreamProperties} // Type of criteria to use for finding gaps public final static String TIME_ONLY_GAP = "timeGap"; public final static String SEQ_ONLY_GAP = "seqGap"; public final static String TIME_SEQ_GAP = "timeSeqGap"; // The method of specifying the gap public final static String SERVICE_CALCULATED = "serviceCalculated"; public final static String USER_SPECIFIED = "userSpecified"; getDataStreamProperties( Long deviceID, // The ID of the Device to get the properties for Long recordType, // The RecordType that will be singled out (devices // can send out more than one RecordType) // 0 = Metadata Packets // 1+ = Device specific record types Boolean checkForGaps, // A Boolean that indicates if the caller wants to // have the service check for data gaps (true means // the service will check for gaps and false/null // means it will not Date startGapCheckWindow, // The start date of the window over which to search for // gaps. Date endGapCheckWindow, // The end date of the window over which to search for gaps. String typeOfGap, // One of three types: "timeGap", "seqGap", "timeSeqGap" Long marginMillis, // This is the number of milliseconds that are used as 'slop' // around the specification for a gap. In other words, if // this is > 0, the service will consider any time between // samples that is less than the specified gap plus this margin, // it will assume that it is not a gap condition. This is to // prevent false positives when the sample timestamps aren't exactly // on the interval. String gapSpec, // There are two ways to specify a gap: // "serviceCalculated" or "userSpecified" Long numberOfRecords, // If the call specifies SERVICE_CALCULATED and this // is greater than 0, the service will use 'numberOfRecords' // most recent records of the specified RecordType // in calculating the average time between samples Date intervalCalcStartWindow, // This is the date that starts the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this is ignored. Date intervalCalcEndWindow, // This is the date that ends the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this will be used as // the endtime and then the service will use the numberOfRecords // before this time as the data to calculate the average // time interval. Long gapInMillis // If the gapSpec is USER_SPECIFIED, then the service will use // this number of milliseconds as the gap for identifying gaps. ) {code} With a return that has the format of: Properties Objects with properties: ||Property Name||Value|| |lastPacketDateTime|This is the date and time of the last packet received| |totalNumberOfRecords|This is the total number of records for the parameters specified| |numberOfFuturePackets|This is the number of packet that appear in the future. This should be zero and if they are not, there could be bad data| |averagSampleIntervalInMillis|This is the number of milliseconds that the service used to find data gaps| |marginInMillis|This is the number of milliseconds as a margin that the service used to find data gaps| |numRecordsSearchedForGaps|This is the number of records that were searched through while trying to find gaps using the gap criteria| |dataGap1Start|This is the date and time of start of the first possible gap in the data| |dataGap1End|This is the date and time of end of the first possible gap in the data| |.|.| |.|.| |.|.| |dataGapNStart|This is the date and time of start of the Nth possible gap in the data| |dataGapNEnd|This is the date and time of end of the Nth possible gap in the data| h5. Java EJB client Under construction h5. REST client If you want to use the REST-style interface the HTTP call would looks something like: {code} http://new-ssds.mbari.org:8080/servlet/DataAccessServlet?objectToInvokeOn=SQLDataStreamRawDataAccess&method=getDataStreamProperties &p1Type=Long&p1Value=1300 &p2Type=Long&p2Value=1 &p3Type=Boolean&p3Value=true &p4Type=Date&p4Value=2008-12-00T00:00:00Z &p5Type=Date&p5Value=2008-01-00T00:00:00Z &p6Type=String&p6Value=timeGap &p7Type=Long&p7Value=5000 &p8Type=String&p8Value=userSpecified &p9Type=Long&p9Value=10 &p10Type=Date&p10Value= &p11Type=Date&p11Value= &p12Type=Long&p12Value=10000 {code} And the return might look something like {code} java.util.Properties{} {code} h5. Web Service Client Under construction {panel}]]> 8912925 846 849 8945769 8913006 10389148 10356436 8945773 8913010 8945777 8913014 8945775 8913012 10389123 10356411 8945782 8913019 8945779 8913016 8945780 8913017 10389127 10356415 10389129 10356417 10389132 10356420 8945790 8913027 8945792 New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/flex and choose for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 8913029 8945736 DataProducer that must be of type "deployment" and then creates "deep" copy of it and persists the new copy. It pulls in the options specified in the incoming parameters to create a unique copy of the DataProducers and DataContainer (outputs). All the rest of the object should be linked to the ones that already exist in the peristent storage. The new ID of the duplicate deployment is returned. h5. The Interface ||Return||Parameters|| |The ID of the newly generated deployment|* DataProducer to Copy * Date the copy should start at * A boolean to indicate if the old (original) should be closed * Date the old (original) should use as an end date (if to be closed) * String that will be the name of the new deployment (copy) * String that is the base of the data stream URI which should be {code}http://new-ssds.mbari.org/servlet/GetOriginalDataServlet{code}| So the API interface looks something like: {code:title=createDuplicateDeepDeployment} public Long createDuplicateDeepDeployment( DataProducer deploymentToCopy, // DataProducer to copy Date newStartDate, // Date to start copied DataProducer on boolean closeOld, // Whether or not old (original) should be closed Date oldEndDate, // Date to use for end date if original is closed String newHeadDeploymentName, // Name of the new DataProducer (copy) String baseDataStreamUri // Base URI for raw data access (http://new-ssds.mbari.org/servlet/GetOriginalDataServlet) ) {code} h5. Java EJB client Under construction h5. REST Style client To use this service via HTTP (REST Style), the call would be constructed using this format: {code:title=REST Style format} http://new-ssds.mbari.org/servlet/MetadataAccessServlet?objectToInvokeOn=DataProducerAccess&method=createDuplicateDeepDeployment &p1Type=DataProducer&p1Value=DataProducer|id=1938292 &p2Type=Date&p2Value=2008-12-01T00:00:00Z &p3Type=boolean&p3Value=true &p4Type=Date&p4Value=2008-11-30T00:00:00Z &p5Type=String&p5Value=New copy of Deployment &p6Type=String&p6Value=http://new-ssds.mbari.org/servlet/GetOriginalDataServlet {code} And the return might looks something like this: {code} Result(s): java.lang.Long|29985 {code} h5. Web Services client Under construction {panel} {panel:title=getDataStreamProperties} h5. Background The desire it to have a service that can characterize a DataStream from an instrument. This would allow for easier monitoring of instruments on the network. h5. The Interface The things that the user would want to know are: ||Return||Parameters|| |Date and time of last packet received|* Device ID * RecordType to search for (nothing/default means most recent packet)| |Total Number Of Records|* Device ID * RecordType to search for (nothing/default specified means all packets)| |Data Gaps|* Device ID * RecordType * Time window over data to search for gaps * Gap criteria ** Type of gap *** Time only *** Sequence number only *** Time and sequence number ** Ways to specify gap *** Margin on gap in milliseconds (anything longer than gap + margin will be considered a possible gap) *** Let service calculate gap constraints (calculate average time between sample) **** Number of points to use (points back from most recent packet) **** Or time window to use (start to end time) *** Specify gap constraints **** Gap in milliseconds| So the API interface looks like: {code:title=getDataStreamProperties} // Type of criteria to use for finding gaps public final static String TIME_ONLY_GAP = "timeGap"; public final static String SEQ_ONLY_GAP = "seqGap"; public final static String TIME_SEQ_GAP = "timeSeqGap"; // The method of specifying the gap public final static String SERVICE_CALCULATED = "serviceCalculated"; public final static String USER_SPECIFIED = "userSpecified"; getDataStreamProperties( Long deviceID, // The ID of the Device to get the properties for Long recordType, // The RecordType that will be singled out (devices // can send out more than one RecordType) // 0 = Metadata Packets // 1+ = Device specific record types Boolean checkForGaps, // A Boolean that indicates if the caller wants to // have the service check for data gaps (true means // the service will check for gaps and false/null // means it will not Date startGapCheckWindow, // The start date of the window over which to search for // gaps. Date endGapCheckWindow, // The end date of the window over which to search for gaps. String typeOfGap, // One of three types: "timeGap", "seqGap", "timeSeqGap" Long marginMillis, // This is the number of milliseconds that are used as 'slop' // around the specification for a gap. In other words, if // this is > 0, the service will consider any time between // samples that is less than the specified gap plus this margin, // it will assume that it is not a gap condition. This is to // prevent false positives when the sample timestamps aren't exactly // on the interval. String gapSpec, // There are two ways to specify a gap: // "serviceCalculated" or "userSpecified" Long numberOfRecords, // If the call specifies SERVICE_CALCULATED and this // is greater than 0, the service will use 'numberOfRecords' // most recent records of the specified RecordType // in calculating the average time between samples Date intervalCalcStartWindow, // This is the date that starts the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this is ignored. Date intervalCalcEndWindow, // This is the date that ends the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this will be used as // the endtime and then the service will use the numberOfRecords // before this time as the data to calculate the average // time interval. Long gapInMillis // If the gapSpec is USER_SPECIFIED, then the service will use // this number of milliseconds as the gap for identifying gaps. ) {code} With a return that has the format of: Properties Objects with properties: ||Property Name||Value|| |lastPacketDateTime|This is the date and time of the last packet received| |totalNumberOfRecords|This is the total number of records for the parameters specified| |numberOfFuturePackets|This is the number of packet that appear in the future. This should be zero and if they are not, there could be bad data| |averagSampleIntervalInMillis|This is the number of milliseconds that the service used to find data gaps| |marginInMillis|This is the number of milliseconds as a margin that the service used to find data gaps| |numRecordsSearchedForGaps|This is the number of records that were searched through while trying to find gaps using the gap criteria| |dataGap1Start|This is the date and time of start of the first possible gap in the data| |dataGap1End|This is the date and time of end of the first possible gap in the data| |.|.| |.|.| |.|.| |dataGapNStart|This is the date and time of start of the Nth possible gap in the data| |dataGapNEnd|This is the date and time of end of the Nth possible gap in the data| h5. Java EJB client Under construction h5. REST client If you want to use the REST-style interface the HTTP call would looks something like: {code} http://new-ssds.mbari.org:8080/servlet/DataAccessServlet?objectToInvokeOn=SQLDataStreamRawDataAccess&method=getDataStreamProperties &p1Type=Long&p1Value=1300 &p2Type=Long&p2Value=1 &p3Type=Boolean&p3Value=true &p4Type=Date&p4Value=2008-12-00T00:00:00Z &p5Type=Date&p5Value=2008-01-00T00:00:00Z &p6Type=String&p6Value=timeGap &p7Type=Long&p7Value=5000 &p8Type=String&p8Value=userSpecified &p9Type=Long&p9Value=10 &p10Type=Date&p10Value= &p11Type=Date&p11Value= &p12Type=Long&p12Value=10000 {code} And the return might look something like {code} java.util.Properties{} {code} h5. Web Service Client Under construction {panel}]]> 8912972 11829813 11797060 8945738 0, the service will consider any time between // samples that is less than the specified gap plus this margin, // it will assume that it is not a gap condition. This is to // prevent false positives when the sample timestamps aren't exactly // on the interval. String gapSpec, // There are two ways to specify a gap: // "serviceCalculated" or "userSpecified" Long numberOfRecords, // If the call specifies SERVICE_CALCULATED and this // is greater than 0, the service will use 'numberOfRecords' // most recent records of the specified RecordType // in calculating the average time between samples Date intervalCalcStartWindow, // This is the date that starts the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this is ignored. Date intervalCalcEndWindow, // This is the date that ends the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this will be used as // the endtime and then the service will use the numberOfRecords // before this time as the data to calculate the average // time interval. Long gapInMillis // If the gapSpec is USER_SPECIFIED, then the service will use // this number of milliseconds as the gap for identifying gaps. ) {code} With a return that has the format of: Properties Objects with properties: ||Property Name||Value|| |lastPacketDateTime|This is the date and time of the last packet received| |totalNumberOfRecords|This is the total number of records for the parameters specified| |numberOfFuturePackets|This is the number of packet that appear in the future. This should be zero and if they are not, there could be bad data| |averagSampleIntervalInMillis|This is the number of milliseconds that the service used to find data gaps| |marginInMillis|This is the number of milliseconds as a margin that the service used to find data gaps| |numRecordsSearchedForGaps|This is the number of records that were searched through while trying to find gaps using the gap criteria| |dataGap1Start|This is the date and time of start of the first possible gap in the data| |dataGap1End|This is the date and time of end of the first possible gap in the data| |.|.| |.|.| |.|.| |dataGapNStart|This is the date and time of start of the Nth possible gap in the data| |dataGapNEnd|This is the date and time of end of the Nth possible gap in the data| h5. Java EJB client Under construction h5. REST client If you want to use the REST-style interface the HTTP call would looks something like: {code} http://new-ssds.mbari.org:8080/servlet/DataAccessServlet?objectToInvokeOn=SQLDataStreamRawDataAccess&method=getDataStreamProperties &p1Type=Long&p1Value=1300 &p2Type=Long&p2Value=1 &p3Type=Boolean&p3Value=true &p4Type=Date&p4Value=2008-12-00T00:00:00Z &p5Type=Date&p5Value=2008-01-00T00:00:00Z &p6Type=String&p6Value=timeGap &p7Type=Long&p7Value=5000 &p8Type=String&p8Value=userSpecified &p9Type=Long&p9Value=10 &p10Type=Date&p10Value= &p11Type=Date&p11Value= &p12Type=Long&p12Value=10000 {code} And the return might look something like {code} java.util.Properties{} {code} h5. Web Service Client Under construction {panel}]]> 8912974 8945737 8912973 8945739 8912975 8945742 8912978 8945741 8912977 11829822 11797069 8945743 8912979 8945745 DataProducer that must be of type "deployment" and then creates "deep" copy of it and persists the new copy. It pulls in the options specified in the incoming parameters to create a unique copy of the DataProducers and DataContainer (outputs). All the rest of the object should be linked to the ones that already exist in the peristent storage. The new ID of the duplicate deployment is returned. h5. The Interface ||Return||Parameters|| |The ID of the newly generated deployment|* DataProducer to Copy * Date the copy should start at * A boolean to indicate if the old (original) should be closed * Date the old (original) should use as an end date (if to be closed) * String that will be the name of the new deployment (copy) * String that is the base of the data stream URI which should be {code}http://new-ssds.mbari.org/servlet/GetOriginalDataServlet{code}| So the API interface looks something like: {code:title=createDuplicateDeepDeployment} public Long createDuplicateDeepDeployment( DataProducer deploymentToCopy, // DataProducer to copy Date newStartDate, // Date to start copied DataProducer on boolean closeOld, // Whether or not old (original) should be closed Date oldEndDate, // Date to use for end date if original is closed String newHeadDeploymentName, // Name of the new DataProducer (copy) String baseDataStreamUri // Base URI for raw data access (http://new-ssds.mbari.org/servlet/GetOriginalDataServlet) ) {code} h5. Java EJB client Under construction h5. REST Style client To use this service via HTTP (REST Style), the call would be constructed using this format: {code:title=REST Style format} http://new-ssds.mbari.org/servlet/MetadataAccessServlet?objectToInvokeOn=DataProducerAccess&method=createDuplicateDeepDeployment &p1Type=DataProducer&p1Value=DataProducer|id=1938292 &p2Type=Date&p2Value=2008-12-01T00:00:00Z &p3Type=boolean&p3Value=true &p4Type=Date&p4Value=2008-11-30T00:00:00Z &p5Type=String&p5Value=New copy of Deployment &p6Type=String&p6Value=http://new-ssds.mbari.org/servlet/GetOriginalDataServlet {code} And the return might looks something like this: {code} Result(s): java.lang.Long|29985 {code} h5. Web Services client Under construction {panel} ]]> 8912981 10389158 10356446 8945747 8912983 8945749 DataProducer that must be of type "deployment" and then creates "deep" copy of it and persists the new copy. It pulls in the options specified in the incoming parameters to create a unique copy of the DataProducers and DataContainer (outputs). All the rest of the object should be linked to the ones that already exist in the peristent storage. The new ID of the duplicate deployment is returned. h5. The Interface ||Return||Parameters|| |The ID of the newly generated deployment|* DataProducer to Copy * Date the copy should start at * A boolean to indicate if the old (original) should be closed * Date the old (original) should use as an end date (if to be closed) * String that will be the name of the new deployment (copy) * String that is the base of the data stream URI which should be {code}http://new-ssds.mbari.org/servlet/GetOriginalDataServlet{code}| So the API interface looks something like: {code:title=createDuplicateDeepDeployment} public Long createDuplicateDeepDeployment( DataProducer deploymentToCopy, // DataProducer to copy Date newStartDate, // Date to start copied DataProducer on boolean closeOld, // Whether or not old (original) should be closed Date oldEndDate, // Date to use for end date if original is closed String newHeadDeploymentName, // Name of the new DataProducer (copy) String baseDataStreamUri // Base URI for raw data access (http://new-ssds.mbari.org/servlet/GetOriginalDataServlet) ) {code} h5. Java EJB client Under construction h5. REST Style client To use this service via HTTP (REST Style), the call would be constructed using this format: {code:title=REST Style format} http://new-ssds.mbari.org/servlet/MetadataAccessServlet?objectToInvokeOn=DataProducerAccess&method=createDuplicateDeepDeployment &p1Type=DataProducer&p1Value=DataProducer|id=1938292 &p2Type=Date&p2Value=2008-12-01T00:00:00Z &p3Type=boolean&p3Value=true &p4Type=Date&p4Value=2008-11-30T00:00:00Z &p5Type=String&p5Value=New copy of Deployment &p6Type=String&p6Value=http://new-ssds.mbari.org/servlet/GetOriginalDataServlet {code} And the return might looks something like this: {code} Result(s): java.lang.Long|29985 {code} h5. Web Services client Under construction {panel} ]]> 8912985 10389165 10356453 11829807 11797054 10389167 10356455 10389080 {code} # It's recommended that the final XML is to be checked into the puckxml module in MBARI's CVS on moonjelly. # Submit the Metadata to SSDS using the SSDSLoads application (available at [http://new-ssds.mbari.org/ssds-docs/client/]. The \-h option provides a usage note): {code} java -jar ssdsLoads-new-ssds.jar -d 1696.xml -x # Use '-x' to test and then '-s' to submit {code} ## If a mistake is made in the metadata (e.g. forgetting the platform Deployment) it may be easier to undo the submission and start over - do a deep delete on the parent deployment if this is the case. ## Minor attribute fixes may be done by editing the database once the deployments have been loaded # Check that the metadata has been successfully loaded using the Explorer application. h2. Establish data publishing application # This step assumes that there will be some application reading data from the deployed instrument. Perhaps it is a legacy application which reads the data to load into a custom data storage or visualization system. For this application to publish data to SSDS it must have visibility of each record the instrument produces as that record needs to be "packaged" into a SensorDataPacket and "published" to SSDS. # An example Java application that will package and publish a record is below: {code} import java.io.IOException; import moos.ssds.jms.PublisherComponent; import moos.ssds.transmogrify.SSDSDevicePacket; /** *

* Publish instrument data records to the SSDS database. The client must provide * SSDS device ID, timeStamp, sequence number, and payload. *

*
* * @author : $Author: mccann $ * @version : $Revision: 1.17.2.7 $ *
*

* The * Monterey Bay Aquarium Research Institute (MBARI) provides this * documentation and code "as is", with no warranty, express * or implied, of its quality or consistency. It is provided without * support and without obligation on the part of MBARI to assist in its * use, correction, modification, or enhancement. This information * should not be published or distributed to third parties without * specific written permission from MBARI. *

*
* Copyright 2008 MBARI.
* MBARI Proprietary Information. All rights reserved.
*
*
*/ /** * @author mccann * */ public class SsdsPublisher { /** * Publish data record from an instrument to SSDS * * @param deviceID * is the SSDS Device ID for the instrument that produces the data in payload * @param epochMilliseconds * time of payload sample in milliseconds since 1/1/1970 0000 GMT * @param sequenceNumber * an incrementing number for each packet * @param payload * data from instrument */ public static void publish(long deviceID, long epochMilliseconds, long sequenceNumber, String payload) { // Create a publisher component PublisherComponent pc = new PublisherComponent(); // Create a new SensorDataPacket SSDSDevicePacket packetToSend = new SSDSDevicePacket(deviceID, payload .getBytes().length); // Set the packet type to data (0 = Metadata, 1 = Data, 2 = Message) packetToSend.setPacketType(1); // Assign the time packetToSend.setSystemTime(epochMilliseconds); // Set the parentID to 0 for a parentless deployment packetToSend.setPlatformID(0); // Set the metadataref number to 0, if the data format changes and we can // refer to a different metadata packet then this number will change. packetToSend.setMetadataRef(0); // Set the sequence number packetToSend.setSequenceNo(sequenceNumber); // Set the payload packetToSend.setDataBuffer(payload.getBytes()); // Set the record type to 1 as this is the most common situation packetToSend.setRecordType(1); try { pc.publishBytes(SSDSDevicePacket .convertToPublishableByteArray(packetToSend)); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } /** * Test of SsdsPublisher.publish() * * @param args * No arguments are taken. Test values hard coded. */ public static void main(String[] args) { /* * Example packet for EITS instrument */ long eitsDeviceID = 1696; // 1696 is actual SSDS deviceID for EITS long sampleTime = 1228431283209L; // Sample time: 2008-12-04 22:54:43 long sampleSequenceNumber = 1; // May set it to ID from DB insert // Payload values must match the RecordVariables in SSDS String samplePayload = "12.2, 4.3, 768.2, 2.3, 1.2, 0.2"; // Temp, Cond, Pres, U, V, W SsdsPublisher.publish(eitsDeviceID, sampleTime, sampleSequenceNumber, samplePayload); } } {code} # An example of a shell script (the getM1 OASIS download) calling a perl script [^ssdsSubmit.pl] to publish download statistics records to SSDS:   {code} # Record start of download set starttime_es = `perl -e 'print time, "\n"'` # Download the data and record error status $DOWNLOAD $DOWNLOAD_OPTS >& $DATAFILE set rtnsts = $status # Record the end of download set endtime_es = `perl -e 'print time, "\n"'` # Get the raw data filesize set filesize = `ls -l $DATAFILE | sed -f $BIN/fs.sed` # Log download statistics and submit to SSDS - IDs are unique for mooring deployment set deviceId = 1698 set parentId = 1306 /oasis/bin/ssdsSubmit.pl $deviceId $parentId "$starttime_es,$endtime_es,$filesize,$rtnsts" {code} # If using Java incorporate the above class into a Java application that reads the data from the instrument and calls a method like the main() example for each record # Set up the application to execute the loads for the duration of the deployment # Configure DStoNetCDF.pl cron job execution as is done for the OASIS data processing within the ssdsadmin account on elvis. (For MARS deployments this is done for the DataProducerGroup MARS2008; see the processCurrentMooringDataStreams.sh script in /u/ssdsadmin/dev/DPforSSDS/cimt/). \\ h2. Generate a packet from a file and publish to SSDS For SIAM packet, you need the following information: # StreamID (short) # DevicePacketVersion (long) # SourceID (long) # Timestamp (long) # SequenceNumber (long) # MetadataRef(long) # ParentID (long) # RecordType (long) # SecondStreamID (short) # FirstBuffer (string that is name of file) # SecondBuffer (string that is name of file) For SSDS packet (ingest) {noformat} # This file contains the information that will be used to construct a data packet to publish to SSDS. # This is the ID of the device that is the source of the packet (transmogrify & ingest) SourceID=1000 # This is the ID of the parent that the device is connected to when it publishes ParentID=999 {noformat}]]> 10356367 10389081 {code} # It's recommended that the final XML is to be checked into the puckxml module in MBARI's CVS on moonjelly. # Submit the Metadata to SSDS using the SSDSLoads application (available at [http://new-ssds.mbari.org/ssds-docs/client/]. The \-h option provides a usage note): {code} java -jar ssdsLoads-new-ssds.jar -d 1696.xml -x # Use '-x' to test and then '-s' to submit {code} ## If a mistake is made in the metadata (e.g. forgetting the platform Deployment) it may be easier to undo the submission and start over - do a deep delete on the parent deployment if this is the case. ## Minor attribute fixes may be done by editing the database once the deployments have been loaded # Check that the metadata has been successfully loaded using the Explorer application. h2. Establish data publishing application # This step assumes that there will be some application reading data from the deployed instrument. Perhaps it is a legacy application which reads the data to load into a custom data storage or visualization system. For this application to publish data to SSDS it must have visibility of each record the instrument produces as that record needs to be "packaged" into a SensorDataPacket and "published" to SSDS. # An example Java application that will package and publish a record is below: {code} import java.io.IOException; import moos.ssds.jms.PublisherComponent; import moos.ssds.transmogrify.SSDSDevicePacket; /** *

* Publish instrument data records to the SSDS database. The client must provide * SSDS device ID, timeStamp, sequence number, and payload. *

*
* * @author : $Author: mccann $ * @version : $Revision: 1.17.2.7 $ *
*

* The * Monterey Bay Aquarium Research Institute (MBARI) provides this * documentation and code "as is", with no warranty, express * or implied, of its quality or consistency. It is provided without * support and without obligation on the part of MBARI to assist in its * use, correction, modification, or enhancement. This information * should not be published or distributed to third parties without * specific written permission from MBARI. *

*
* Copyright 2008 MBARI.
* MBARI Proprietary Information. All rights reserved.
*
*
*/ /** * @author mccann * */ public class SsdsPublisher { /** * Publish data record from an instrument to SSDS * * @param deviceID * is the SSDS Device ID for the instrument that produces the data in payload * @param epochMilliseconds * time of payload sample in milliseconds since 1/1/1970 0000 GMT * @param sequenceNumber * an incrementing number for each packet * @param payload * data from instrument */ public static void publish(long deviceID, long epochMilliseconds, long sequenceNumber, String payload) { // Create a publisher component PublisherComponent pc = new PublisherComponent(); // Create a new SensorDataPacket SSDSDevicePacket packetToSend = new SSDSDevicePacket(deviceID, payload .getBytes().length); // Set the packet type to data (0 = Metadata, 1 = Data, 2 = Message) packetToSend.setPacketType(1); // Assign the time packetToSend.setSystemTime(epochMilliseconds); // Set the parentID to 0 for a parentless deployment packetToSend.setPlatformID(0); // Set the metadataref number to 0, if the data format changes and we can // refer to a different metadata packet then this number will change. packetToSend.setMetadataRef(0); // Set the sequence number packetToSend.setSequenceNo(sequenceNumber); // Set the payload packetToSend.setDataBuffer(payload.getBytes()); // Set the record type to 1 as this is the most common situation packetToSend.setRecordType(1); try { pc.publishBytes(SSDSDevicePacket .convertToPublishableByteArray(packetToSend)); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } /** * Test of SsdsPublisher.publish() * * @param args * No arguments are taken. Test values hard coded. */ public static void main(String[] args) { /* * Example packet for EITS instrument */ long eitsDeviceID = 1696; // 1696 is actual SSDS deviceID for EITS long sampleTime = 1228431283209L; // Sample time: 2008-12-04 22:54:43 long sampleSequenceNumber = 1; // May set it to ID from DB insert // Payload values must match the RecordVariables in SSDS String samplePayload = "12.2, 4.3, 768.2, 2.3, 1.2, 0.2"; // Temp, Cond, Pres, U, V, W SsdsPublisher.publish(eitsDeviceID, sampleTime, sampleSequenceNumber, samplePayload); } } {code} # An example of a shell script (the getM1 OASIS download) calling a perl script [^ssdsSubmit.pl] to publish download statistics records to SSDS:   {code} # Record start of download set starttime_es = `perl -e 'print time, "\n"'` # Download the data and record error status $DOWNLOAD $DOWNLOAD_OPTS >& $DATAFILE set rtnsts = $status # Record the end of download set endtime_es = `perl -e 'print time, "\n"'` # Get the raw data filesize set filesize = `ls -l $DATAFILE | sed -f $BIN/fs.sed` # Log download statistics and submit to SSDS - IDs are unique for mooring deployment set deviceId = 1698 set parentId = 1306 /oasis/bin/ssdsSubmit.pl $deviceId $parentId "$starttime_es,$endtime_es,$filesize,$rtnsts" {code} # If using Java incorporate the above class into a Java application that reads the data from the instrument and calls a method like the main() example for each record # Set up the application to execute the loads for the duration of the deployment # Configure DStoNetCDF.pl cron job execution as is done for the OASIS data processing within the ssdsadmin account on elvis. (For MARS deployments this is done for the DataProducerGroup MARS2008; see the processCurrentMooringDataStreams.sh script in /u/ssdsadmin/dev/DPforSSDS/cimt/). \\ h2. Generate a packet from a file and publish to SSDS {warning:title=Under Construction} This section is still under construction, pay no attention {warning} # Download a jar file # Create the properties file like the one here: {noformat} # This file contains the information that will be used to construct a data packet to publish to SSDS. # This is a SIAM property that (in theory) would allow multiple message formats # that can be handled. Right now, there is only one and it should always be 0. DevicePacketVersion=0 # This is the ID of the source of the data (i.e. the device that generated the packet # being sent. SourceID=1 # This is the timestamp on the packet which normally represents when the packet # was created. In the system it is normally epoch seconds, but for clarity sake, # you can enter it in ISO 8601 format (http://en.wikipedia.org/wiki/ISO_8601) Timestamp=2009-09-30T15:47:00Z # This is the sequence number that will be on the packet. It should reflect the # index in the order of generation of packets from the source device. SequenceNumber=1 # This is the reference number that points to the sequence number of the packet # which contains the metadata that describes the contents of this packet. MetadataRef=0 # This is the ID of the parent device (if one exists) that the generating device # (see SourceID above) was connected to when it generated the packet ParentID=0 # This is the type of record that is being produced. Source devices can produce # multiple types of records of the same type. For example, a device can produce # data records in multiple formats. This allows you to identify which format # is used for the generated packet. It serves to link the pre-defined metadata # to the packet and allows machines and humans to understand the contents of the # packet. RecordType=0 # This value determines what type of packet (SIAM equivalent) will be constructed # using the information in this file. The available options are: # 1. METADATA # 2. SENSOR_DATA # 3. DEVICE_MESSAGE SecondStreamID=METADATA # This indicates which version of the above type of packet will be constructed. # At the time of this authoring, the only option available is 0 for all three # types listed above. In theory this would allow you to have multiple forms of # any of the above types of packets, but it has not been used to date. SecondPacketVersion=0 # This is the name of the file (located with this properties file) that contains # the payload of the first buffer. It will be read as straight bytes into a # byte buffer. NOTE: with METADATA packets the first buffer contains something # that is called 'cause' which is not used much by SSDS. FirstBuffer=first-buffer.xml # This is the name of the file (located with this properties file) that contains # the payload of the second buffer. It will be read as straight bytes into a # byte buffer. NOTE: with METADATA packets the second buffer is usually where the # main metadata is stored (for example, device XML). SecondBuffer=second-buffer.txt {noformat} # Create the two buffer files # Run using command: {noformat} the command {noformat}]]> 10356368 10389078 {code} # It's recommended that the final XML is to be checked into the puckxml module in MBARI's CVS on moonjelly. # Submit the Metadata to SSDS using the SSDSLoads application (available at [http://new-ssds.mbari.org/ssds-docs/client/]. The \-h option provides a usage note): {code} java -jar ssdsLoads-new-ssds.jar -d 1696.xml -x # Use '-x' to test and then '-s' to submit {code} ## If a mistake is made in the metadata (e.g. forgetting the platform Deployment) it may be easier to undo the submission and start over - do a deep delete on the parent deployment if this is the case. ## Minor attribute fixes may be done by editing the database once the deployments have been loaded # Check that the metadata has been successfully loaded using the Explorer application. h2. Establish data publishing application # This step assumes that there will be some application reading data from the deployed instrument. Perhaps it is a legacy application which reads the data to load into a custom data storage or visualization system. For this application to publish data to SSDS it must have visibility of each record the instrument produces as that record needs to be "packaged" into a SensorDataPacket and "published" to SSDS. # An example Java application that will package and publish a record is below: {code} import java.io.IOException; import moos.ssds.jms.PublisherComponent; import moos.ssds.transmogrify.SSDSDevicePacket; /** *

* Publish instrument data records to the SSDS database. The client must provide * SSDS device ID, timeStamp, sequence number, and payload. *

*
* * @author : $Author: mccann $ * @version : $Revision: 1.17.2.7 $ *
*

* The * Monterey Bay Aquarium Research Institute (MBARI) provides this * documentation and code "as is", with no warranty, express * or implied, of its quality or consistency. It is provided without * support and without obligation on the part of MBARI to assist in its * use, correction, modification, or enhancement. This information * should not be published or distributed to third parties without * specific written permission from MBARI. *

*
* Copyright 2008 MBARI.
* MBARI Proprietary Information. All rights reserved.
*
*
*/ /** * @author mccann * */ public class SsdsPublisher { /** * Publish data record from an instrument to SSDS * * @param deviceID * is the SSDS Device ID for the instrument that produces the data in payload * @param epochMilliseconds * time of payload sample in milliseconds since 1/1/1970 0000 GMT * @param sequenceNumber * an incrementing number for each packet * @param payload * data from instrument */ public static void publish(long deviceID, long epochMilliseconds, long sequenceNumber, String payload) { // Create a publisher component PublisherComponent pc = new PublisherComponent(); // Create a new SensorDataPacket SSDSDevicePacket packetToSend = new SSDSDevicePacket(deviceID, payload .getBytes().length); // Set the packet type to data (0 = Metadata, 1 = Data, 2 = Message) packetToSend.setPacketType(1); // Assign the time packetToSend.setSystemTime(epochMilliseconds); // Set the parentID to 0 for a parentless deployment packetToSend.setPlatformID(0); // Set the metadataref number to 0, if the data format changes and we can // refer to a different metadata packet then this number will change. packetToSend.setMetadataRef(0); // Set the sequence number packetToSend.setSequenceNo(sequenceNumber); // Set the payload packetToSend.setDataBuffer(payload.getBytes()); // Set the record type to 1 as this is the most common situation packetToSend.setRecordType(1); try { pc.publishBytes(SSDSDevicePacket .convertToPublishableByteArray(packetToSend)); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } /** * Test of SsdsPublisher.publish() * * @param args * No arguments are taken. Test values hard coded. */ public static void main(String[] args) { /* * Example packet for EITS instrument */ long eitsDeviceID = 1696; // 1696 is actual SSDS deviceID for EITS long sampleTime = 1228431283209L; // Sample time: 2008-12-04 22:54:43 long sampleSequenceNumber = 1; // May set it to ID from DB insert // Payload values must match the RecordVariables in SSDS String samplePayload = "12.2, 4.3, 768.2, 2.3, 1.2, 0.2"; // Temp, Cond, Pres, U, V, W SsdsPublisher.publish(eitsDeviceID, sampleTime, sampleSequenceNumber, samplePayload); } } {code} # An example of a shell script (the getM1 OASIS download) calling a perl script [^ssdsSubmit.pl] to publish download statistics records to SSDS:   {code} # Record start of download set starttime_es = `perl -e 'print time, "\n"'` # Download the data and record error status $DOWNLOAD $DOWNLOAD_OPTS >& $DATAFILE set rtnsts = $status # Record the end of download set endtime_es = `perl -e 'print time, "\n"'` # Get the raw data filesize set filesize = `ls -l $DATAFILE | sed -f $BIN/fs.sed` # Log download statistics and submit to SSDS - IDs are unique for mooring deployment set deviceId = 1698 set parentId = 1306 /oasis/bin/ssdsSubmit.pl $deviceId $parentId "$starttime_es,$endtime_es,$filesize,$rtnsts" {code} # If using Java incorporate the above class into a Java application that reads the data from the instrument and calls a method like the main() example for each record # Set up the application to execute the loads for the duration of the deployment # Configure DStoNetCDF.pl cron job execution as is done for the OASIS data processing within the ssdsadmin account on elvis. (For MARS deployments this is done for the DataProducerGroup MARS2008; see the processCurrentMooringDataStreams.sh script in /u/ssdsadmin/dev/DPforSSDS/cimt/). \\ h2. Generate a packet from a file and publish to SSDS For SIAM packet (transmogrify), you need the following information: # StreamID (short) # DevicePacketVersion (long) # SourceID (long) # Timestamp (long) # SequenceNumber (long) # MetadataRef(long) # ParentID (long) # RecordType (long) # SecondStreamID (short) # FirstBufferLength (int) # FirstBuffer (byte []) # SecondBufferLength (int) # SecondBuffer (byte []) For SSDS packet (ingest) {noformat} # This file contains the information that will be used to construct a data packet to publish to SSDS. # There are two points you can publish data to (transmogrify|ingest): # 1. Transmogrify in the SIAM form (transmogrify) # 2. Ingest in the SSDS form (ingest) publish.location=transmogrify # This is the ID of the device that is the source of the packet (transmogrify & ingest) device.id=1000 # This is the ID of the parent that the device is connected to when it publishes parent.id=999 {noformat}]]> 10356365 10389071 {code} # It's recommended that the final XML is to be checked into the puckxml module in MBARI's CVS on moonjelly. # Submit the Metadata to SSDS using the SSDSLoads application (available at [http://new-ssds.mbari.org/ssds-docs/client/]. The \-h option provides a usage note): {code} java -jar ssdsLoads-new-ssds.jar -d 1696.xml -x # Use '-x' to test and then '-s' to submit {code} ## If a mistake is made in the metadata (e.g. forgetting the platform Deployment) it may be easier to undo the submission and start over - do a deep delete on the parent deployment if this is the case. ## Minor attribute fixes may be done by editing the database once the deployments have been loaded # Check that the metadata has been successfully loaded using the Explorer application. h2. Establish data publishing application # This step assumes that there will be some application reading data from the deployed instrument. Perhaps it is a legacy application which reads the data to load into a custom data storage or visualization system. For this application to publish data to SSDS it must have visibility of each record the instrument produces as that record needs to be "packaged" into a SensorDataPacket and "published" to SSDS. # An example Java application that will package and publish a record is below: {code} import java.io.IOException; import moos.ssds.jms.PublisherComponent; import moos.ssds.transmogrify.SSDSDevicePacket; /** *

* Publish instrument data records to the SSDS database. The client must provide * SSDS device ID, timeStamp, sequence number, and payload. *

*
* * @author : $Author: mccann $ * @version : $Revision: 1.17.2.7 $ *
*

* The * Monterey Bay Aquarium Research Institute (MBARI) provides this * documentation and code "as is", with no warranty, express * or implied, of its quality or consistency. It is provided without * support and without obligation on the part of MBARI to assist in its * use, correction, modification, or enhancement. This information * should not be published or distributed to third parties without * specific written permission from MBARI. *

*
* Copyright 2008 MBARI.
* MBARI Proprietary Information. All rights reserved.
*
*
*/ /** * @author mccann * */ public class SsdsPublisher { /** * Publish data record from an instrument to SSDS * * @param deviceID * is the SSDS Device ID for the instrument that produces the data in payload * @param epochMilliseconds * time of payload sample in milliseconds since 1/1/1970 0000 GMT * @param sequenceNumber * an incrementing number for each packet * @param payload * data from instrument */ public static void publish(long deviceID, long epochMilliseconds, long sequenceNumber, String payload) { // Create a publisher component PublisherComponent pc = new PublisherComponent(); // Create a new SensorDataPacket SSDSDevicePacket packetToSend = new SSDSDevicePacket(deviceID, payload .getBytes().length); // Set the packet type to data (0 = Metadata, 1 = Data, 2 = Message) packetToSend.setPacketType(1); // Assign the time packetToSend.setSystemTime(epochMilliseconds); // Set the parentID to 0 for a parentless deployment packetToSend.setPlatformID(0); // Set the metadataref number to 0, if the data format changes and we can // refer to a different metadata packet then this number will change. packetToSend.setMetadataRef(0); // Set the sequence number packetToSend.setSequenceNo(sequenceNumber); // Set the payload packetToSend.setDataBuffer(payload.getBytes()); // Set the record type to 1 as this is the most common situation packetToSend.setRecordType(1); try { pc.publishBytes(SSDSDevicePacket .convertToPublishableByteArray(packetToSend)); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } /** * Test of SsdsPublisher.publish() * * @param args * No arguments are taken. Test values hard coded. */ public static void main(String[] args) { /* * Example packet for EITS instrument */ long eitsDeviceID = 1696; // 1696 is actual SSDS deviceID for EITS long sampleTime = 1228431283209L; // Sample time: 2008-12-04 22:54:43 long sampleSequenceNumber = 1; // May set it to ID from DB insert // Payload values must match the RecordVariables in SSDS String samplePayload = "12.2, 4.3, 768.2, 2.3, 1.2, 0.2"; // Temp, Cond, Pres, U, V, W SsdsPublisher.publish(eitsDeviceID, sampleTime, sampleSequenceNumber, samplePayload); } } {code} # An example of a shell script (the getM1 OASIS download) calling a perl script [^ssdsSubmit.pl] to publish download statistics records to SSDS:   {code} # Record start of download set starttime_es = `perl -e 'print time, "\n"'` # Download the data and record error status $DOWNLOAD $DOWNLOAD_OPTS >& $DATAFILE set rtnsts = $status # Record the end of download set endtime_es = `perl -e 'print time, "\n"'` # Get the raw data filesize set filesize = `ls -l $DATAFILE | sed -f $BIN/fs.sed` # Log download statistics and submit to SSDS - IDs are unique for mooring deployment set deviceId = 1698 set parentId = 1306 /oasis/bin/ssdsSubmit.pl $deviceId $parentId "$starttime_es,$endtime_es,$filesize,$rtnsts" {code} # If using Java incorporate the above class into a Java application that reads the data from the instrument and calls a method like the main() example for each record # Set up the application to execute the loads for the duration of the deployment # Configure DStoNetCDF.pl cron job execution as is done for the OASIS data processing within the ssdsadmin account on elvis. (For MARS deployments this is done for the DataProducerGroup MARS2008; see the processCurrentMooringDataStreams.sh script in /u/ssdsadmin/dev/DPforSSDS/cimt/). \\]]> 10356358 10389074 {code} # It's recommended that the final XML is to be checked into the puckxml module in MBARI's CVS on moonjelly. # Submit the Metadata to SSDS using the SSDSLoads application (available at [http://new-ssds.mbari.org/ssds-docs/client/]. The \-h option provides a usage note): {code} java -jar ssdsLoads-new-ssds.jar -d 1696.xml -x # Use '-x' to test and then '-s' to submit {code} ## If a mistake is made in the metadata (e.g. forgetting the platform Deployment) it may be easier to undo the submission and start over - do a deep delete on the parent deployment if this is the case. ## Minor attribute fixes may be done by editing the database once the deployments have been loaded # Check that the metadata has been successfully loaded using the Explorer application. h2. Establish data publishing application # This step assumes that there will be some application reading data from the deployed instrument. Perhaps it is a legacy application which reads the data to load into a custom data storage or visualization system. For this application to publish data to SSDS it must have visibility of each record the instrument produces as that record needs to be "packaged" into a SensorDataPacket and "published" to SSDS. # An example Java application that will package and publish a record is below: {code} import java.io.IOException; import moos.ssds.jms.PublisherComponent; import moos.ssds.transmogrify.SSDSDevicePacket; /** *

* Publish instrument data records to the SSDS database. The client must provide * SSDS device ID, timeStamp, sequence number, and payload. *

*
* * @author : $Author: mccann $ * @version : $Revision: 1.17.2.7 $ *
*

* The * Monterey Bay Aquarium Research Institute (MBARI) provides this * documentation and code "as is", with no warranty, express * or implied, of its quality or consistency. It is provided without * support and without obligation on the part of MBARI to assist in its * use, correction, modification, or enhancement. This information * should not be published or distributed to third parties without * specific written permission from MBARI. *

*
* Copyright 2008 MBARI.
* MBARI Proprietary Information. All rights reserved.
*
*
*/ /** * @author mccann * */ public class SsdsPublisher { /** * Publish data record from an instrument to SSDS * * @param deviceID * is the SSDS Device ID for the instrument that produces the data in payload * @param epochMilliseconds * time of payload sample in milliseconds since 1/1/1970 0000 GMT * @param sequenceNumber * an incrementing number for each packet * @param payload * data from instrument */ public static void publish(long deviceID, long epochMilliseconds, long sequenceNumber, String payload) { // Create a publisher component PublisherComponent pc = new PublisherComponent(); // Create a new SensorDataPacket SSDSDevicePacket packetToSend = new SSDSDevicePacket(deviceID, payload .getBytes().length); // Set the packet type to data (0 = Metadata, 1 = Data, 2 = Message) packetToSend.setPacketType(1); // Assign the time packetToSend.setSystemTime(epochMilliseconds); // Set the parentID to 0 for a parentless deployment packetToSend.setPlatformID(0); // Set the metadataref number to 0, if the data format changes and we can // refer to a different metadata packet then this number will change. packetToSend.setMetadataRef(0); // Set the sequence number packetToSend.setSequenceNo(sequenceNumber); // Set the payload packetToSend.setDataBuffer(payload.getBytes()); // Set the record type to 1 as this is the most common situation packetToSend.setRecordType(1); try { pc.publishBytes(SSDSDevicePacket .convertToPublishableByteArray(packetToSend)); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } /** * Test of SsdsPublisher.publish() * * @param args * No arguments are taken. Test values hard coded. */ public static void main(String[] args) { /* * Example packet for EITS instrument */ long eitsDeviceID = 1696; // 1696 is actual SSDS deviceID for EITS long sampleTime = 1228431283209L; // Sample time: 2008-12-04 22:54:43 long sampleSequenceNumber = 1; // May set it to ID from DB insert // Payload values must match the RecordVariables in SSDS String samplePayload = "12.2, 4.3, 768.2, 2.3, 1.2, 0.2"; // Temp, Cond, Pres, U, V, W SsdsPublisher.publish(eitsDeviceID, sampleTime, sampleSequenceNumber, samplePayload); } } {code} # An example of a shell script (the getM1 OASIS download) calling a perl script [^ssdsSubmit.pl] to publish download statistics records to SSDS:   {code} # Record start of download set starttime_es = `perl -e 'print time, "\n"'` # Download the data and record error status $DOWNLOAD $DOWNLOAD_OPTS >& $DATAFILE set rtnsts = $status # Record the end of download set endtime_es = `perl -e 'print time, "\n"'` # Get the raw data filesize set filesize = `ls -l $DATAFILE | sed -f $BIN/fs.sed` # Log download statistics and submit to SSDS - IDs are unique for mooring deployment set deviceId = 1698 set parentId = 1306 /oasis/bin/ssdsSubmit.pl $deviceId $parentId "$starttime_es,$endtime_es,$filesize,$rtnsts" {code} # If using Java incorporate the above class into a Java application that reads the data from the instrument and calls a method like the main() example for each record # Set up the application to execute the loads for the duration of the deployment # Configure DStoNetCDF.pl cron job execution as is done for the OASIS data processing within the ssdsadmin account on elvis. (For MARS deployments this is done for the DataProducerGroup MARS2008; see the processCurrentMooringDataStreams.sh script in /u/ssdsadmin/dev/DPforSSDS/cimt/). \\]]> 10356361 10389065 New->Other...' and then select 'Server->Server'. Then 'Next>' # Then select 'JBoss->JBoss v4.2'. The default server host of 'localhost' and server name of 'JBoss v4.2 on localhost' is fine. Click on 'Next>'. # You can use the Default JRE and for the 'Application Server Directory' browse to the location where you unpacked JBoss.]]> 10356352 11830013 11797269 11830012 11797268 11830017 metadataRef # metadataRef->metadataSequenceNumber # metadataRef->dataDescriptionID | # metadataRef # metadataSequenceNumber # dataDescriptionID | | parentId | direct copy to both parentID and platformID | # parentId # platformID | | recordType | If DevicePacket is MetadataPacket, set recordType to 0, otherwise, set to recordtype of DevicePacket | recordType | | | If MetadataPacket, packetType = 0, if SensorDataPacket, packetType = 1, if DeviceMessagePacket, packetType = 2 | packetType | | firstBufferLength | ignored | | | firstBuffer | First buffer depends on which type of packet. If MetadataPacket, copy "bytes" buffer, if SensorDataPacket, copy "dataBuffer", if DeviceMessagePacket, copy "message" | firstBuffer | | secondBufferLength | ignored | | | secondBuffer | Only exists if MetadataPacket and will copy over "cause" buffer | secondBuffer | h5. Exploration of Upgrade of Ingest/Transmogrify to AMQP In an effort to allow non-Java clients to send data to SSDS in the form of messages and to upgrade the messaging system to a technology that is more scalable and higher performance, an investigation of AMQP implementations was done. # [Qpid Exploration] ]]> 11797273 11830015 11797271 11830005 11797261 11830006 11797262 11830010 11797266 11830008 11797264 11830002 11797258 11830000 11797256 11829999 11797255 15664050 15631300 11829903 11797155 11829905 11797157 11829899 11797151 11829901 11797153 11829897 11797149 15664052 15631302 8945860 New->Other.. # Select Flex Builder->Flex Project # Type in 'ssds-flex' for name # Uncheck 'Use default location' # Browse to SSDS_HOME/src/web and select choose # Select 'Web Application' # Select 'J2EE' as Application Server Type # Check Use remote object access service # Click Next> # Uncheck 'Use default location for Local LifeCycle Data Service server' # Browse to SSDS_HOME/src/resources/flex and choose for the 'Root folder' # Change 'Root URL' to the URL of the servlet context (for example, on localhost it would be 'http://localhost:8080/servlet/') # Change the 'Context Root'to '/servlet/' # Select 'Compile Application Locally in Flex Builder' # For 'Output folder location' put your deployment directory for the web application (for example /Users/kgomes/Applications/jboss-4.2.2.GA/server/default/deploy/ssds.war) # Click Validate Configuration (it will warn that the output folder is not a subfolder of the server root (that is OK). {note:title=Will create a Main.mxml} Note that the creation of the Flex Builder project will create a Main.mxml file. To fix this, right click on explorer.mxml and choose 'Set As Default Application', then you can delete the Main.mxml file. {note} NOTE: To run the tests fully, you must have perl installed with the following module: # Class-ObjectTemplate-0.7 (http://search.cpan.org/~jasons/Class-ObjectTemplate-0.7/) ]]> 8913100 10945019 10912264 15664154 metadataRef # metadataRef->metadataSequenceNumber # metadataRef->dataDescriptionID | # metadataRef # metadataSequenceNumber # dataDescriptionID | | parentId | direct copy: # parentId->parentId # parentId->platformID | # parentId # platformID | | recordType | # MetadataPacket: recordType to 0 # Other: direct copy | recordType | | | # If MetadataPacket, packetType = 0 # If SensorDataPacket, packetType = 1 # If DeviceMessagePacket, packetType = 2 | packetType | | firstBufferLength | ignored | | | firstBuffer | First buffer depends on which type of packet # If MetadataPacket, copy "bytes" buffer # If SensorDataPacket, copy "dataBuffer" # If DeviceMessagePacket, copy "message" | firstBuffer | | secondBufferLength | ignored | | | secondBuffer | Only exists if MetadataPacket and will copy over "cause" buffer | secondBuffer | h6. DevicePacket to SIAM Byte Array This is done by the SIAM Exportable Packet class || DevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataRef | direct copy | metadataRef | | parentId | direct copy | parentId | | recordType | direct copy | recordType | | | This is set based on what type of packet: # If MetadataPacket, set to 0x0101 # If SensorDataPacket, set to 0x0102 # If DeviceMessagePacket, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | firstBufferLength | direct copy (see note for first buffer) | firstBufferLength | | firstBuffer | This depends on the type of packet: # If MetadataPacket, "cause" bytes are copied over # If SensorDataPacket, "dataMessage" bytes are copied over # If DeviceMessagePacket, "message" bytes are copied over | firstBuffer | | secondBufferLength | only valid with MetadataPacket, but is copied directly over | secondBufferLength | | secondBuffer | only valid with MetadataPacket, and the "buffer" bytes are copied over | secondBuffer | h6. SSDSDevicePacket to SIAM Byte Array *Originally done in TransmogrifyMDB by calling SSDSDevicePacket.convertToPublishableVersion3ByteArray before passing byte array to method to translate to SSDS format and then send to ingest* || SSDSDevicePacket || Translation Rule || SIAM Byte Array || | | This is a static value that is set to indicate the byte array is a DevicePacket and is set to 0x0100 | EX_DEVICEPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | sourceID | direct copy | sourceID | | systemTime | direct copy | systemTime | | sequenceNo | direct copy | sequenceNo | | metadataSequenceNumber | direct copy | metadataRef | | parentID | direct copy | parentID | | recordType | # If packetType = 0, set recordType = 0 # If packetType = 1, set recordType = recordType # If packetType = 2, set recordType = recordType | recordType | | | This is set based on what type of packet: # If packetType = 0, set to 0x0101 # If packetType = 1, set to 0x0102 # If packetType = 2, set to 0x0103 | EX_XXXXXXPACKET | | | This is just the serial version UID of the class which is always 0 | serialVersionUID | | | This depends on the type of packet: # If packetType = 0, set to length of "otherBuffer" # If packetType = 1, set to length of "dataBuffer" # If packetType = 2, set to length of "dataBuffer" | firstBufferLength | | other/dataBuffer | This depends on the type of packet: # If packetType = 0, set to bytes from "otherBuffer" # If packetType = 1, set to bytes from "dataBuffer" # If packetType = 2, set to bytes from "dataBuffer" | firstBuffer | | | This only exists if it is packetType = 0 then it is set to the length of the "dataBuffer" | secondBufferLength | | dataBuffer | This only exists if it is packetType = 0 then it is set to the byte from the "dataBuffer" | secondBuffer | h5. SSDSDevicePacket to SSDS Byte Array *Originally done in SSDSDevicePacket.convertToVersion3ByteArray* || SSDSDevicePacket || Translation Rule || SSDS Byte Array || | sourceID | direct copy | sourceID | | systemTime | ignored during the translation directly, but used through getter methods for seconds and nanoseconds | | | timestampSeconds | direct copy (note that this is _sort of_ a direct copy, there are getter methods on SSDSDevicePacket that convert the systemTime to seconds and nanoseconds when called). | timestampSeconds | | timestampNanoseconds | direct copy (note that this is _sort of_ a direct copy, there are getter methods on SSDSDevicePacket that convert the systemTime to seconds and nanoseconds when called). | timestampNanoseconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | ignored | | | parentID | ignored | | | recordType | If packetType = 0, set packetSubType to 0, otherwise set to recordType | packetSubType | | packetType | Depends on packetType: # If packetType = 0, set to 1 # If packetType = 1, set to 0 # If packetType = 2, set to 4 | packetType | | metadataSequenceNumber | direct copy | metadataSequenceNumber | | dataDescriptionVersion | direct copy | dataDescriptionVersion | | platformID | direct copy | parentID | | | copy length of dataBuffer | firstBufferLength | | dataBuffer | direct copy | firstBuffer | | | copy length of otherBuffer | secondBufferLength | | otherBuffer | direct copy | secondBuffer | h6. SIAM Byte Array to SSDS Byte Array *Originally done in TransmogrifyMDB in checkAndPublishBytes method* || SIAM Byte Array || Translation Rules || SSDS Byte Array || | EX_DEVICEPACKET | ignored | | | serialVersionUID | ignored | | | sourceID | direct copy | sourceID | | | Depending on EX_XXXXXXXPACKET: # If MetadataPacket, set packetType to 1 # If SensorDataPacket, set packetType to 0 # If DeviceMessagePacket, set packetType to 4 | packetType | | | This was set using the SIAMMetadataTracker that tried to keep track of real version numbers based on XML in payload | metadataSequencNumber | | systemTime | Split into timestampSeconds and timestampNanoseconds | # timestampSeconds # timestampNanoSeconds | | sequenceNo | direct copy | sequenceNumber | | metadataRef | direct copy | dataDescriptionVersion | | parentId | direct copy | parentID | | recordType | If MetadataPacket (determined from EX_XXXXXXXPACKET), recordType set to 0, otherwise set to recordType | packetSubType | | EX_XXXXXXXPACKET | ignored in storage, but used in logic | | | serialVersionUID | ignored | | | first/secondBufferLength | If MetadataPacket (determined from EX_XXXXXXXPACKET), firstBufferLength is set to secondBufferLength so we can flip the "cause" and "buffer" bytes because it just made more sense since the cause was rarely populated. Otherwise set to firstBufferLength | firstBufferLength | | first/secondBuffer | If MetadataPacket (determined from EX_XXXXXXXPACKET), firstBuffer is set to secondBuffer so we can flip the "cause" and "buffer" bytes because it just made more sense since the cause was rarely populated. Otherwise set to firstBuffer | firstBuffer | | firstBufferLength | If MetadataPacket (determined from EX_XXXXXXXPACKET), secondBufferLength is set to firstBufferLength to flip "cause" and "buffer" bytes | secondBufferLength | | firstBuffer | If MetadataPacket (determined from EX_XXXXXXXPACKET), secondBuffer is set to firstBuffer to flip "cause" and "buffer" bytes | secondBuffer | h5. Exploration of Upgrade of Ingest/Transmogrify to AMQP In an effort to allow non-Java clients to send data to SSDS in the form of messages and to upgrade the messaging system to a technology that is more scalable and higher performance, an investigation of AMQP implementations was done. # [Qpid Exploration] ]]> 15631403 1047 1050 10322085 1 10355672 2009-08-07 07:06:15.083 2009-08-07 07:06:15.083 15237240 15204476 4948030 15 {noformat} to {noformat} 1 {noformat} Which should effectively make the RuminateMDB a singleton which should alleviate our deadlock issues that we were having (at least at Ruminate step). {note}]]> 4915267 4948020 4915257 4948022 15 {noformat} to {noformat} 1 {noformat} Which should effectively make the RuminateMDB a singleton which should alleviate our deadlock issues that we were having (at least at Ruminate step). {note}]]> 4915259 4948018 4915255 3179956 3114484 10387742 tag. For instance any nominalLat/Lon/Depth. {color:#ff0000}The one exception is the nominalDepth, if it is known please set it{color}. # If the tag has a 'name' attribute, make sure it does not have any deployment specific information in it. For example, 'ISUS Deployment' is better than 'M2 ISUS Deployment'. The reason for removing any deployment information from the XML is so that when the device moves to a different mooring, the user should not have to edit the XML. The goal is to get all the XML to a point where it never needs to be edited when an instrument is deployed (unless something in the way the data stream is generated from the instrument changes). # Go to the SSDS Device pages and verify that the all the device information (mfg, model, serial number, name, type, etc.) matches what is currently in SSDS. If any of those are different it will update the device information in SSDS when the XML comes in the data stream. # Verify RecordDescription and RecordVariables look correct. I usually go to the raw data pages in SSDS and bring up the last few packets from the device just to verify that the number of columns and bufferSeparator look about right. # Check any changes to the XML back into CVS. # Copy the XML to the \\Tornado\ssdsdata\mooring\(m1\|m2)\YYYY\xml directory # Go to the \\Tornado\ssdsdata\mooring(m1\|m2)\YYYY\cfg directory. # This next step is the one that needs to be timed with the mooring turn. When the old instrument is shutdown: ## Open the ssds.cfg file in a text editor ## Find the line that shows the currently deployed instrument and copy it to a line just below it. For example, if we are replacing the GPS, it might look like this before: {panel:title=Before Copy} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} and this after: {panel:title=After Copy} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # Now change the new line to have the {color:#cc0000}{+}correct device ID{+}{color} *and* the {color:#cc0000}{+}correct XML{+}{color} file URL {panel:title=After Device ID update} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,,,TransformGPS instrument = GPS_TYPE3,1511,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1511.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} # To clean up the previous deployment information, put start and end dates after the XML URL {panel:title=After Adding Start/End dates} {noformat} instrument = PCO2,1471,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1471.xml instrument = Metsys,1480,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1480.xml instrument = GPS_TYPE3,1416,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1416.xml,2007/04/25 16:21:58,2007/08/01 10:00:00,TransformGPS instrument = GPS_TYPE3,1511,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1511.xml,,,TransformGPS instrument = Spec_PRR,1420,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1420.xml instrument = ADCP,1417,http://dods.mbari.org/data/ssdsdata/mooring/m2/2007/xml/1417.xml {noformat} {panel} (Be careful on the format for the start and end date strings.  Leading 0s are required.) # Save the cfg file. {note:title=Saving the file will make the change take hold}When the ssds.cfg file changes (saved) is when the OASIS2SSDS processing will pick up the instrument change. Now, the next time it runs it will pick up the instrument change, grab the XML file from the 'xml' directory and publish it to SSDS. It will then publish all data under the new device ID. {note} # Edit the metadata in SSDS to put a close date on the old instrument deployment in SSDS. Currently I do that using Enterprise Manager. h5. If this is being done after the fact the next steps will also need to be taken. # After the new deployment shows up in SSDS (which can take up to 5 minutes after the OASIS2SSDS has completed), the start time for the new deployment will need to be edited to match the actual time the instrument was swapped. # Also because the data was being published under the incorrect device ID, it will need to be moved from one database table in SSDS_Data on Solstice to another table. ## The first thing that I do is grab the timestamp from the last packet sent from the old device in the raw data page on SSDS. ## For example, I go to: [http://new-ssds.mbari.org:8080/ssds/siamRawDataStep1.jsp] and enter the old device ID and set the number of packets back to make sure it goes far enough back to cover the actual time of the instrument swap. Then click on 'Next->'. ## Once the raw data shows up, find the last packet from the old device and grab the 'SIAM Timestamp' value (not the date/time) as that will be used in the Enterprise Manager query. ## Open Enterprise Manager and navigate to the 'SSDS_Data' database on Solstice. ## Browse the tables and find the table with the device ID of the old device and right click on it and select 'Open Table->Return all rows'. ## Click on the 'SQL' button in Enterprise Manager to bring up the SQL pane. It should show the basic query which should look something like this: {noformat} SELECT * FROM [1416] {noformat} ## Now add the where clause to pick only the data that is after the timestamp you grabbed from the last packet on the web page. {note:title=Timestamps in SQL are in Seconds}A quick note here, the 'SIAM Timestamp' on the raw data page is actually in milliseconds and the database column is in seconds so you will have to remove the last three digits of the 'SIAM Timestamp' before putting it in this query. {note} {noformat} SELECT * FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ## Run this query by clicking the run button '\!' in Enterprise Manager. ## Look over the results to make sure they look about right (usually you are looking for the length of the return which should be much shorter). You can actually use a count query to see how many rows this query will return. A count query would look like: {noformat} SELECT count(*) FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ## Once you know the query is correct (also compare sequence number in query return and the raw data page), copy it to the clipboard and close the query window in Enterprise Manager. ## Navigate to the table of the device you want to copy the data into and right click and select 'All Tasks->Import Data...' which will fire up the DTS wizard. ### Click on 'Next>' ### For the Data Source database choose Solstice ### Select the 'SSDS_Data' database (note you should have permissions to do all this and use your windows authentication) ### Click on 'Next>' ### The destination configuration should be already to go (Solistice and SSDS_Data database). ### Click on 'Next>' ### Select 'Use a query to specify the data to transfer' ### Click on 'Next>' ### Paste the query from your clipboard into the 'Query Statement' window (You can click on 'Parse' if you want a quick sanity check) ### Click on 'Next>' ### Click on the 'Results' entry under the 'Destination' column which will enable a drop down box. ### Choose the table of the newly installed device where you will be copying the data to. ### Click on 'Next>' ### Click on 'Next>' ### Click on 'Finish' which will copy the data. ### Once that is done, open the table of the old instrument and the SQL pane so that we can construct the delete query on the old data. ### Paste in the select query and verify it is the same data you copied over: {noformat} SELECT * FROM [1416] WHERE timestampSeconds > 1185963023 {noformat} ### If it looks good, click on the 'Change Query type ...' button in Enterprise Manager and select 'Delete'. This will change the query to a delete query. ### Run the query by click on the run '\!' button. That will remove all the data from the old instrument. That's it ... whew\! Kevin Gomes (August 3, 2007)]]> 10354987 10944587 10911821 10944588 10911822 10944590 10911824 10944593 10911828 10944594 0, the service will consider any time between // samples that is less than the specified gap plus this margin, // it will assume that it is not a gap condition. This is to // prevent false positives when the sample timestamps aren't exactly // on the interval. String gapSpec, // There are two ways to specify a gap: // "serviceCalculated" or "userSpecified" Long numberOfRecords, // If the call specifies SERVICE_CALCULATED and this // is greater than 0, the service will use 'numberOfRecords' // most recent records of the specified RecordType // in calculating the average time between samples Date intervalCalcStartWindow, // This is the date that starts the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this is ignored. Date intervalCalcEndWindow, // This is the date that ends the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this will be used as // the endtime and then the service will use the numberOfRecords // before this time as the data to calculate the average // time interval. Long gapInMillis // If the gapSpec is USER_SPECIFIED, then the service will use // this number of milliseconds as the gap for identifying gaps. ) {code} With a return that has the format of: Properties Objects with properties: ||Property Name||Value|| |lastPacketDateTime|This is the date and time of the last packet received| |totalNumberOfRecords|This is the total number of records for the parameters specified| |numberOfFuturePackets|This is the number of packet that appear in the future. This should be zero and if they are not, there could be bad data| |averagSampleIntervalInMillis|This is the number of milliseconds that the service used to find data gaps| |marginInMillis|This is the number of milliseconds as a margin that the service used to find data gaps| |numRecordsSearchedForGaps|This is the number of records that were searched through while trying to find gaps using the gap criteria| |dataGap1Start|This is the date and time of start of the first possible gap in the data| |dataGap1End|This is the date and time of end of the first possible gap in the data| |.|.| |.|.| |.|.| |dataGapNStart|This is the date and time of start of the Nth possible gap in the data| |dataGapNEnd|This is the date and time of end of the Nth possible gap in the data| h5. Java EJB client Under construction h5. REST client If you want to use the REST-style interface the HTTP call would looks something like: {code} http://new-ssds.mbari.org:8080/servlet/DataAccessServlet?objectToInvokeOn=SQLDataStreamRawDataAccess&method=getDataStreamProperties &p1Type=Long&p1Value=1300 &p2Type=Long&p2Value=1 &p3Type=Boolean&p3Value=true &p4Type=Date&p4Value=2008-12-00T00:00:00Z &p5Type=Date&p5Value=2008-01-00T00:00:00Z &p6Type=String&p6Value=timeGap &p7Type=Long&p7Value=5000 &p8Type=String&p8Value=userSpecified &p9Type=Long&p9Value=10 &p10Type=Date&p10Value= &p11Type=Date&p11Value= &p12Type=Long&p12Value=10000 {code} And the return might look something like {code} java.util.Properties{} {code} h5. Web Service Client Under construction {panel}]]> 10911829 10944584 10911817 10944604 10911840 10944603 10911839 10944606 10911842 10944605 10911841 10944608 10911844 10944607 0, the service will consider any time between // samples that is less than the specified gap plus this margin, // it will assume that it is not a gap condition. This is to // prevent false positives when the sample timestamps aren't exactly // on the interval. String gapSpec, // There are two ways to specify a gap: // "serviceCalculated" or "userSpecified" Long numberOfRecords, // If the call specifies SERVICE_CALCULATED and this // is greater than 0, the service will use 'numberOfRecords' // most recent records of the specified RecordType // in calculating the average time between samples Date intervalCalcStartWindow, // This is the date that starts the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this is ignored. Date intervalCalcEndWindow, // This is the date that ends the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this will be used as // the endtime and then the service will use the numberOfRecords // before this time as the data to calculate the average // time interval. Long gapInMillis // If the gapSpec is USER_SPECIFIED, then the service will use // this number of milliseconds as the gap for identifying gaps. ) {code} With a return that has the format of: Properties Objects with properties: ||Property Name||Value|| |lastPacketDateTime|This is the date and time of the last packet received| |totalNumberOfRecords|This is the total number of records for the parameters specified| |numberOfFuturePackets|This is the number of packet that appear in the future. This should be zero and if they are not, there could be bad data| |averagSampleIntervalInMillis|This is the number of milliseconds that the service used to find data gaps| |marginInMillis|This is the number of milliseconds as a margin that the service used to find data gaps| |numRecordsSearchedForGaps|This is the number of records that were searched through while trying to find gaps using the gap criteria| |dataGap1Start|This is the date and time of start of the first possible gap in the data| |dataGap1End|This is the date and time of end of the first possible gap in the data| |.|.| |.|.| |.|.| |dataGapNStart|This is the date and time of start of the Nth possible gap in the data| |dataGapNEnd|This is the date and time of end of the Nth possible gap in the data| h5. Java EJB client Under construction h5. REST client If you want to use the REST-style interface the HTTP call would looks something like: {code} http://new-ssds.mbari.org:8080/servlet/DataAccessServlet?objectToInvokeOn=SQLDataStreamRawDataAccess&method=getDataStreamProperties &p1Type=Long&p1Value=1300 &p2Type=Long&p2Value=1 &p3Type=Boolean&p3Value=true &p4Type=Date&p4Value=2008-12-00T00:00:00Z &p5Type=Date&p5Value=2008-01-00T00:00:00Z &p6Type=String&p6Value=timeGap &p7Type=Long&p7Value=5000 &p8Type=String&p8Value=userSpecified &p9Type=Long&p9Value=10 &p10Type=Date&p10Value= &p11Type=Date&p11Value= &p12Type=Long&p12Value=10000 {code} And the return might look something like {code} java.util.Properties{} {code} h5. Web Service Client Under construction {panel}]]> 10911843 10944610 10911846 10944596 10911831 10944595 0, the service will consider any time between // samples that is less than the specified gap plus this margin, // it will assume that it is not a gap condition. This is to // prevent false positives when the sample timestamps aren't exactly // on the interval. String gapSpec, // There are two ways to specify a gap: // "serviceCalculated" or "userSpecified" Long numberOfRecords, // If the call specifies SERVICE_CALCULATED and this // is greater than 0, the service will use 'numberOfRecords' // most recent records of the specified RecordType // in calculating the average time between samples Date intervalCalcStartWindow, // This is the date that starts the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this is ignored. Date intervalCalcEndWindow, // This is the date that ends the window over which the data // will be used to calculate the average time between samples // NOTE: If numberOfRecords is specified, this will be used as // the endtime and then the service will use the numberOfRecords // before this time as the data to calculate the average // time interval. Long gapInMillis // If the gapSpec is USER_SPECIFIED, then the service will use // this number of milliseconds as the gap for identifying gaps. ) {code} With a return that has the format of: Properties Objects with properties: ||Property Name||Value|| |lastPacketDateTime|This is the date and time of the last packet received| |totalNumberOfRecords|This is the total number of records for the parameters specified| |numberOfFuturePackets|This is the number of packet that appear in the future. This should be zero and if they are not, there could be bad data| |averagSampleIntervalInMillis|This is the number of milliseconds that the service used to find data gaps| |marginInMillis|This is the number of milliseconds as a margin that the service used to find data gaps| |numRecordsSearchedForGaps|This is the number of records that were searched through while trying to find gaps using the gap criteria| |dataGap1Start|This is the date and time of start of the first possible gap in the data| |dataGap1End|This is the date and time of end of the first possible gap in the data| |.|.| |.|.| |.|.| |dataGapNStart|This is the date and time of start of the Nth possible gap in the data| |dataGapNEnd|This is the date and time of end of the Nth possible gap in the data| h5. Java EJB client Under construction h5. REST client If you want to use the REST-style interface the HTTP call would looks something like: {code} http://new-ssds.mbari.org:8080/servlet/DataAccessServlet?objectToInvokeOn=SQLDataStreamRawDataAccess&method=getDataStreamProperties &p1Type=Long&p1Value=1300 &p2Type=Long&p2Value=1 &p3Type=Boolean&p3Value=true &p4Type=Date&p4Value=2008-12-00T00:00:00Z &p5Type=Date&p5Value=2008-01-00T00:00:00Z &p6Type=String&p6Value=timeGap &p7Type=Long&p7Value=5000 &p8Type=String&p8Value=userSpecified &p9Type=Long&p9Value=10 &p10Type=Date&p10Value= &p11Type=Date&p11Value= &p12Type=Long&p12Value=10000 {code} And the return might look something like {code} java.util.Properties{} {code} h5. Web Service Client Under construction {panel}]]> 10911830 10944598 10911833 10944599 10911834 9830523 9797754 10944601 10911837 10944613 10911849 10944614 10911850 10944617 10911853 10944615 10911851 10944635 10911871 10944627 10911863 10944633 10911869 13369347 13336579 9830448 9797679 9830451 9797682 9830453 9797684 9830456 9797687 9830455 9797686 9830460 9797691 10944575 10911808 10944574 10911807 10944573 10911806 4489265 4456499 16 18 2 2006-10-17 21:48:40.373 2026-02-04 08:33:05.020 8912969 5 false 0.0 0 0 8912971 5 false 0.0 0 0 10 6 false 0.0 0 0 10911836 6 false 0.0 0 0 4456506 5 false 0.0 0 0 10911848 6 false 0.0 0 0 1179738 6 false 0.0 0 0 17858877 5 false 0.0 0 0 10911819 6 false 0.0 0 0 81 5 false 0.0 0 0 8355981 6 false 0.0 0 0 80 5 false 0.0 0 0 8061037 6 false 0.0 0 0 8912969 6 false 0.0 0 0 841 5 false 0.0 0 0 8912971 6 false 0.0 0 0 13926583 5 false 0.0 0 0 17858877 6 false 0.0 0 0 4456506 6 false 0.0 0 0 10912280 5 false 0.0 0 0 9797689 5 false 0.0 0 0 80 6 false 0.0 0 0 10356008 5 false 0.0 0 0 8355981 5 false 0.0 0 0 81 6 false 0.0 0 0 5374184 5 false 0.0 0 0 91 6 false 0.0 0 0 10911819 5 false 0.0 0 0 3637922 5 false 0.0 0 0 1179945 5 false 0.0 0 0 10 5 false 0.0 0 0 5374184 6 false 0.0 0 0 10356008 6 false 0.0 0 0 8356090 6 false 0.0 0 0 1179945 6 false 0.0 0 0 13664436 5 false 0.0 0 0 841 6 false 0.0 0 0 9797681 5 false 0.0 0 0 10912280 6 false 0.0 0 0 3637922 6 false 0.0 0 0 11797071 5 false 0.0 0 0 1179738 5 false 0.0 0 0 11797073 5 false 0.0 0 0 17236005 5 false 0.0 0 0 11797058 6 false 0.0 0 0 11797058 5 false 0.0 0 0 8355863 6 false 0.0 0 0 8061037 5 false 0.0 0 0 9797681 6 false 0.0 0 0 3637846 6 false 0.0 0 0 17236005 6 false 0.0 0 0 10911848 5 false 0.0 0 0 13926583 6 false 0.0 0 0 9797689 6 false 0.0 0 0 10911836 5 false 0.0 0 0 10356413 5 false 0.0 0 0 10356413 6 false 0.0 0 0 8356090 5 false 0.0 0 0 10355890 5 false 0.0 0 0 8355863 5 false 0.0 0 0 11797071 6 false 0.0 0 0 10355664 6 false 0.0 0 0 10355664 5 false 0.0 0 0 11240414 6 false 0.0 0 0 11797073 6 false 0.0 0 0 11240414 5 false 0.0 0 0 10355890 6 false 0.0 0 0 91 5 false 0.0 0 0 13664436 6 false 0.0 0 0 3637846 5 false 0.0 0 0