8913211 1179693 1179693 1409026 8945967 6 2009-02-04 08:24:54.600 2019-01-16 13:14:31.410 8913213 9371988 11239968 16416904 31326212 1179926 1179693 1179693 1409026 1212691 1247023 1247024 1247025 1247026 1247027 1247028 1247029 1247030 1247031 1247032 1247033 1247034 1247035 1247036 1247037 1247038 1247039 1247040 1247041 1933326 7 2007-08-01 12:00:41.590 2007-08-01 12:30:07.993 1179927 1179928 1179931 1179932 1179933 1179934 8913216 1179693 8913263 1179693 1409026 8945970 47 2009-02-04 09:06:26.687 2019-10-30 13:01:50.153 8913219 8913220 8913221 8913261 8913267 9371687 9371689 9371690 9371691 9371842 9371843 9371845 9371846 9371847 9371849 9371851 9371852 9371854 9371856 9371968 9371976 9371977 10912221 16416906 16416910 16777271 16777273 16777275 19041454 29196532 29196534 29196536 29196537 29196538 29196540 29687875 29687877 29687878 29687879 29819111 30703904 30703906 31326214 31326216 31555589 32178179 9371999 1179693 1179693 1409026 9404748 11 2009-03-03 08:49:09.297 2012-12-17 13:58:14.453 9372001 9372014 9372040 9372059 9372060 9372074 9372087 9372098 9372145 19041456 1409026 1179692 1179693 4882497 4882498 4882499 4882500 4882501 4882502 4882503 19300409 34406557 34406558 34406559 34406560 34406561 34406562 34406563 34406564 34406565 34406566 34406567 34406568 34406569 34406570 2007-06-08 14:58:06.287 2009-02-05 23:30:16.357 global 5374441 1179693 1179693 1409026 5407177 13 2008-06-25 12:52:44.010 2008-06-30 08:33:47.513 5374443 5374467 5374480 5374481 5374483 5374485 5374487 5374489 5374490 5374491 5374499 5374524 5374679 9372076 8913230 1179693 8913230 1409026 9404822 31621186 8 2009-03-05 15:48:36.407 2019-02-25 07:29:03.363 9372140 9372337 9798533 10355079 14516245 19041450 31555586 16777434 1179693 1179693 1409026 16810195 3 2011-05-23 10:13:56.883 2012-12-17 13:35:07.567 19041447 19041448 8913230 1179693 9372076 1179693 1409026 8945983 27656193 27656194 27656195 27656196 27656197 36 2009-02-04 13:47:04.580 2015-07-06 08:21:40.890 8913270 9371961 9371965 9371984 9371986 9371997 9372045 9372047 9372049 9372051 9372071 9372113 9372115 9372141 9372168 9372170 9372177 9797822 9798609 10355081 10912037 10912039 10912046 10912054 14516242 14516243 14516247 16416798 16416800 16416869 19041452 20087082 20087083 22282301 27557907 9502744 9502745 9502770 9502771 16547846 19234817 9371973 1179693 1179693 1409026 9404722 6 2009-03-02 16:47:50.330 2009-03-09 10:43:14.650 9371974 9372023 9372053 9372054 9372117 9371824 1179693 1179693 1409026 9404574 33652737 7 2009-02-23 10:11:31.067 2022-02-17 08:39:10.573 9371828 9371830 9371832 9371834 9371835 33587202 9502732 9502733 9502734 9502735 9502736 8913162 1179693 1179693 1409026 8945919 3 2009-02-02 16:30:29.693 2009-02-04 23:38:46.330 8913163 8913234 9076758 9076763 9076766 9076767 9502922 21528634 21528635 21528636 9371817 1409026 9404567 2 2009-02-23 08:37:11.143 2011-05-18 14:40:46.267 16777416 1835079 1179693 1179693 1409026 1867843 1 2007-10-03 10:53:22.300 2007-10-03 10:53:22.300 21365241 1179693 1179693 21365381 21365395 21365482 21365488 21365523 21365547 21365564 21365583 21365738 1409026 21397973 21433175 21433176 21433177 6 2013-06-27 08:00:20.133 2013-06-27 08:40:33.937 21365245 21365248 21365250 21365252 21365253 21528613 21528614 21528624 8913263 8913216 1179693 8913216 1409026 8946016 3 2009-02-05 16:40:27.063 2009-03-02 16:03:06.713 9371685 9371970 1507333 2]]> 1507332 theme.key ]]> 9372068 1179693 1179693 1409026 9404814 72 2009-03-05 15:41:25.080 2009-07-30 17:28:07.453 9372069 9372081 9372092 9372096 9372100 9372102 9372108 9372110 9372134 9372138 9372160 9372162 9372166 9372172 9372175 9372179 9372182 9372184 9372185 9372187 9372205 9372207 9372217 9372267 9797662 9797758 9797762 9797764 9797765 9797794 9797808 9797817 9797824 9797829 9797830 9797922 9797936 9797949 9797969 9798021 9798289 9798432 9798444 9798449 10354859 10354877 10354879 10354904 10354914 10354915 10354917 10354919 10355011 10355062 10355077 10355084 10355097 10355129 10355187 10355240 10355253 10355279 10355368 10355370 10355371 10355402 10355428 10355514 10355572 10355607 10355650 1507334 CoreData false global ]]> 9372235 1409026 9404973 33292289 33292290 33292291 33292292 31522820 23 2009-03-12 15:26:12.667 2021-06-03 12:47:20.397 9372237 9372239 9798698 11796644 11796646 11796648 21365957 21365959 21365960 21365961 21365963 21365964 21365974 21365975 21365977 21365979 22511642 29819155 31555597 31555599 31555600 33095684 1179693 1179926 1835079 1835085 5374441 8913162 8913167 8913211 8913216 8913230 9371824 9371973 9371999 9372027 9372036 9372068 16777434 20971983 21365241 1409026 1212461 33652738 33652739 33652740 33652741 33652742 33652743 33652744 33652745 33652746 33652747 33652748 33652749 33652750 33652751 33652752 33652753 33652754 33652755 33652756 33652757 33652758 33652759 33652760 33652761 33652762 33652763 33652764 33652765 33652766 33652767 33652768 33652769 33652770 33652771 33652772 33652773 33652774 33652775 33652776 33652777 33652778 33652779 33652780 33652781 33652782 33652783 33652784 33652785 33652786 33652787 33652788 33652789 33652790 33652791 33652792 1343503 1343504 1343505 1343507 1343508 1343509 1933325 1933367 3014678 3276915 3604643 4849686 4849687 9306119 9306120 9306153 19857414 26771530 26771531 31424514 32243713 117 2007-06-08 14:58:06.293 2022-02-17 08:40:35.507 1179694 1179695 1179696 1179697 1179698 1179699 1179700 1179701 1179702 1179703 1179704 1179873 1179893 1179925 1179929 1179930 1179935 1179936 1179937 1179939 1179941 1179942 1835078 1835088 2163068 2163069 2163070 2785317 3113043 3113130 5374439 5374445 5374687 5374725 5374727 5832706 5832734 5832735 5832736 8913159 8913160 8913165 8913192 8913207 8913209 8913212 8913223 8913232 8913233 8913241 8913265 8913328 8913329 8913331 9371683 9371684 9371794 9371795 9371818 9371819 9371820 9371822 9371826 9371837 9371971 9372003 9372005 9372018 9372021 9372024 9372028 9372062 9372064 9372073 9372078 9372082 9372111 9372128 9372129 9372132 9372241 9372408 9372410 10355281 10355284 10355285 10355289 10355291 10356042 10356044 10356046 10356047 11240432 16777429 16777431 16777432 16777435 18251828 19041597 19041606 19041607 19041609 19041610 19041612 19041614 19041615 19041617 19041640 20971975 20971976 26640696 26640699 26640961 31555585 32407555 33587204 1540098 1540099 1540127 1540128 1540130 1540131 1540132 1540133 1540134 1540135 1540136 1540137 1540138 1540139 1540140 1540141 1540142 1540143 1540144 1540145 1540146 1540147 1540148 1540149 1540150 1540151 1540152 1540153 1540154 1540155 1540156 1540157 1540158 1540159 1540160 3309604 3309605 3309606 3309607 5439528 5439534 5767170 5767179 5767182 9076785 9076791 9076800 10485954 11403361 18186261 9076786 19169608 19169612 19169614 19169617 21037074 26837028 26837033 19169610 26837030 21037076 10485953 18841602 9076787 9076789 18186262 11403363 11403364 11403362 10485955 10485889 10485890 9502731 18186260 19169609 21037075 19169615 26837029 9502737 10485952 10485977 10485976 11403360 11403366 11403367 11403368 11403369 11403379 11403380 11763726 16973885 16973886 16973887 16973888 16973889 16973890 16973891 16973892 16973893 16973894 16973895 16973896 16973897 16973898 16973899 16973900 17170433 19169598 19169599 18186263 18841601 19169597 19169616 21037073 26837027 26837031 26837032 9011207 9011208 19234818 9372036 1179693 1179693 1409026 9404782 1 2009-03-04 08:58:49.743 2009-03-04 08:58:49.743 1835085 1179693 1179693 1409026 1867849 1933366 7 2007-10-03 11:50:19.747 2007-10-03 15:50:42.903 1835086 1835087 1835115 1835116 1835117 1835118 8913167 1179693 1179693 1409026 8945924 2 2009-02-03 09:34:08.390 2009-03-03 08:29:09.873 9371989 20971983 1179693 1179693 21365339 21365524 21365582 1409026 21004746 21433404 21433405 21433406 21433407 21433408 21433409 21433410 21433411 21433412 21433413 47 2013-05-06 15:48:48.900 2013-07-03 13:40:54.457 20971985 20971986 20971988 20971989 20971991 20971993 20971995 20972003 20972005 20972006 20972007 20972008 20972009 20972018 20972020 20972022 20972024 20972026 20972027 20972033 20972035 20972042 20972059 20972060 20972062 20972074 21364738 21364739 21364741 21364743 21364745 21364747 21364748 21364750 21365293 21365295 21365297 21365299 21365301 21365303 21365305 21365307 21365309 21365311 21365333 21365335 9372027 1179693 1179693 1409026 9404773 3 2009-03-04 08:19:28.920 2009-03-05 15:24:10.317 9372033 9372065 21365975 21398680 14 2009-03-12 15:26:12.667 2013-08-12 16:00:08.367 9372235 9372267 9405003 24 2009-03-05 15:41:25.080 2009-03-11 17:15:19.143 9372068 21365977 21398682 15 2009-03-12 15:26:12.667 2013-08-12 16:00:47.157 9372235 21365974 21398679 13 2009-03-12 15:26:12.667 2013-08-12 13:40:14.340 9372235 8945983 8913230 19857414 6 1179693 2013-02-04 14:43:15.033 2016-01-19 12:13:15.030 21365963 21398668 11 2009-03-12 15:26:12.667 2013-08-12 13:30:48.180 9372235 21365964 21398669 12 2009-03-12 15:26:12.667 2013-08-12 13:38:50.427 9372235 1247037 1179926 2007-08-01 12:30:08.010 2007-08-01 12:30:08.010 1247036 1179926 2007-08-01 12:30:08.010 2007-08-01 12:30:08.010 1247039 1179926 2007-08-01 12:30:08.010 2007-08-01 12:30:08.010 1247038 1179926 2007-08-01 12:30:08.010 2007-08-01 12:30:08.010 1247041 1179926 2007-08-01 12:30:08.010 2007-08-01 12:30:08.010 1247040 1179926 2007-08-01 12:30:08.010 2007-08-01 12:30:08.010 18251828 18284595 98 2007-06-08 14:58:06.293 2011-05-23 10:14:37.250 1179693 21364748 21397516 33 2013-05-06 15:48:48.900 2013-05-20 10:45:53.203 20971983 1247029 1179926 2007-08-01 12:30:08.010 2007-08-01 12:30:08.010 21364747 21397515 32 2013-05-06 15:48:48.900 2013-05-16 16:28:48.940 20971983 8945967 8913211 1247028 1179926 2007-08-01 12:30:08.010 2007-08-01 12:30:08.010 21364750 21397518 34 2013-05-06 15:48:48.900 2013-05-20 10:46:42.647 20971983 5832706 5865474 36 2007-06-08 14:58:06.293 2008-07-08 11:43:08.467 1179693 8945970 8913216 1247031 1179926 2007-08-01 12:30:08.010 2007-08-01 12:30:08.010 10355607 10388335 70 2009-03-05 15:41:25.080 2009-07-23 17:07:33.457 9372068 1247030 1179926 2007-08-01 12:30:08.010 2007-08-01 12:30:08.010 1247033 1179926 2007-08-01 12:30:08.010 2007-08-01 12:30:08.010 33095684 33161220 22 2009-03-12 15:26:12.667 2019-04-04 16:03:01.020 9372235 1247032 1179926 2007-08-01 12:30:08.010 2007-08-01 12:30:08.010 21365979 21398684 16 2009-03-12 15:26:12.667 2013-08-12 16:13:14.800 9372235 1247035 1179926 2007-08-01 12:30:08.010 2007-08-01 12:30:08.010 1247034 1179926 2007-08-01 12:30:08.010 2007-08-01 12:30:08.010 21364741 21397509 29 2013-05-06 15:48:48.900 2013-05-16 16:21:17.533 20971983 21364739 21397507 28 2013-05-06 15:48:48.900 2013-05-16 16:18:50.300 20971983 21364745 21397513 31 2013-05-06 15:48:48.900 2013-05-16 16:25:59.477 20971983 21364743 21397511 30 2013-05-06 15:48:48.900 2013-05-16 16:22:45.537 20971983 3309606 1179693 2008-01-30 11:20:19.320 2009-02-06 18:01:00.547 1785 1 3309607 1179693 2008-01-30 11:20:19.327 2009-02-06 18:00:40.997 2205 1 3309604 1179693 2008-01-30 11:19:39.237 2008-01-30 11:19:39.237 324867 1 3309605 1179693 2008-01-30 11:20:19.313 2009-02-06 18:01:13.527 1583 1 21364738 21397506 27 2013-05-06 15:48:48.900 2013-05-15 08:36:41.043 20971983 21365961 21398666 10 2009-03-12 15:26:12.667 2013-08-12 13:29:30.763 9372235 21365960 21398665 9 2009-03-12 15:26:12.667 2013-08-12 13:29:06.403 9372235 21365959 21398664 8 2009-03-12 15:26:12.667 2013-08-12 13:27:18.983 9372235 21365957 21398662 7 2009-03-12 15:26:12.667 2010-04-05 11:25:53.157 9372235 8945924 8913167 5832736 5865500 39 2007-06-08 14:58:06.293 2008-07-15 17:23:17.273 1179693 5832735 5865499 38 2007-06-08 14:58:06.293 2008-07-15 17:09:19.093 1179693 5832734 5865498 37 2007-06-08 14:58:06.293 2008-07-14 12:11:57.353 1179693 10355572 10388300 69 2009-03-05 15:41:25.080 2009-07-21 16:16:35.987 9372068 10912221 10944976 23 2009-02-04 09:06:26.687 2009-03-02 16:53:34.180 8913216 20971976 21004739 110 2007-06-08 14:58:06.293 2013-05-03 21:11:33.217 1179693 9372337 9405070 2 2009-03-05 15:48:36.407 2009-03-09 12:17:04.570 9372076 20971975 21004738 109 2007-06-08 14:58:06.293 2013-01-15 16:25:22.040 1179693 21365583 21365241 21398303 21434802 21434803 5 2013-07-23 15:23:20.427 2013-08-07 08:47:07.843 21365584 21365721 21365722 21365769 21365582 20971983 21398302 2 2013-07-23 15:22:05.897 2013-07-23 15:25:04.360 21365585 9306153 1 1179693 2009-03-19 20:29:15.080 2009-03-19 20:29:15.080 31326216 31358984 44 2009-02-04 09:06:26.687 2019-01-16 13:20:48.383 8913216 31326212 31358980 5 2009-02-04 08:24:54.600 2011-03-23 12:27:09.390 8913211 31326214 31358982 43 2009-02-04 09:06:26.687 2018-11-30 11:58:44.220 8913216 21365564 21365241 21398284 21434014 5 2013-07-23 09:42:27.857 2013-07-23 11:29:59.883 21365570 21365571 21365572 21365573 1247026 1179926 2007-08-01 12:30:08.010 2007-08-01 12:30:08.010 20972009 21004772 13 2013-05-06 15:48:48.900 2013-05-07 16:20:46.333 20971983 1247027 1179926 2007-08-01 12:30:08.010 2007-08-01 12:30:08.010 20972008 21004771 12 2013-05-06 15:48:48.900 2013-05-07 16:20:24.607 20971983 1247024 1179926 2007-08-01 12:30:08.010 2007-08-01 12:30:08.010 20972007 21004770 11 2013-05-06 15:48:48.900 2013-05-07 16:19:45.470 20971983 1247025 1179926 2007-08-01 12:30:08.010 2007-08-01 12:30:08.010 20972006 21004769 10 2013-05-06 15:48:48.900 2013-05-07 16:19:02.787 20971983 5374679 5407410 12 2008-06-25 12:52:44.010 2008-06-25 17:00:34.923 5374441 20972005 21004768 9 2013-05-06 15:48:48.900 2013-05-07 16:09:02.423 20971983 1247023 1179926 2007-08-01 12:30:08.010 2007-08-01 12:30:08.010 21365547 21365241 21398267 2 2013-07-19 14:45:12.383 2013-07-19 15:08:38.340 21365548 10355650 10388378 71 2009-03-05 15:41:25.080 2009-07-27 16:59:25.817 9372068 20972003 21004766 8 2013-05-06 15:48:48.900 2013-05-06 16:45:54.027 20971983 1212461 1179693 20971995 21004758 7 2013-05-06 15:48:48.900 2013-05-06 16:40:07.157 20971983 8946016 8913263 20971993 21004756 6 2013-05-06 15:48:48.900 2013-05-06 16:31:25.197 20971983 20971991 21004754 5 2013-05-06 15:48:48.900 2013-05-06 15:56:39.743 20971983 20971989 21004752 4 2013-05-06 15:48:48.900 2013-05-06 15:56:22.290 20971983 20971988 21004751 3 2013-05-06 15:48:48.900 2013-05-06 15:53:18.953 20971983 20971985 21004748 1 2013-05-06 15:48:48.900 2013-05-06 15:48:48.900 20971983 20971986 21004749 2 2013-05-06 15:48:48.900 2013-05-06 15:52:26.617 20971983 21365523 21365241 21398245 21433992 21433993 1 2013-07-17 16:28:55.120 2013-07-17 16:28:55.120 21365524 20971983 21398246 3 2013-07-17 17:16:11.377 2013-07-23 10:43:59.677 21365565 21365566 5374687 5407418 33 2007-06-08 14:58:06.293 2008-06-25 12:56:42.127 1179693 1540135 1179693 2007-08-01 12:42:56.600 2007-08-01 12:42:56.600 6175 1 1540134 1179693 2007-08-01 12:42:20.370 2007-08-01 12:42:20.370 4262 1 31555589 31588357 45 2009-02-04 09:06:26.687 2019-01-17 10:28:38.383 8913216 1540137 1179693 2007-08-01 12:44:46.600 2007-08-01 12:44:46.600 10639 1 34406569 1409026 2026-02-04 08:21:44.129 2026-02-04 08:21:44.129 1540136 1179693 2007-08-01 12:44:23.050 2007-08-01 12:44:23.050 10222 1 34406570 1409026 2026-02-04 08:21:44.129 2026-02-04 08:21:44.129 9798289 9831042 41 2009-03-05 15:41:25.080 2009-04-30 15:01:18.510 9372068 34406567 1409026 2026-02-04 08:21:44.129 2026-02-04 08:21:44.129 1540139 1179693 2007-08-01 12:45:32.153 2007-08-01 12:45:32.153 9760 1 34406568 1409026 2026-02-04 08:21:44.129 2026-02-04 08:21:44.129 10912046 10944805 23 2009-02-04 13:47:04.580 2009-11-11 08:41:47.730 8913230 5374445 5407181 32 2007-06-08 14:58:06.293 2008-06-25 12:47:24.317 1179693 1540138 1179693 2007-08-01 12:45:12.860 2007-08-01 12:45:12.860 10756 1 34406565 1409026 2026-02-04 08:21:44.129 2026-02-04 08:21:44.129 1540141 1179693 2007-08-01 12:46:26.957 2007-08-01 12:46:26.957 9574 1 34406566 1409026 2026-02-04 08:21:44.129 2026-02-04 08:21:44.129 5374443 5407179 1 2008-06-25 12:52:44.010 2008-06-25 12:52:44.010 5374441 1540140 1179693 2007-08-01 12:45:56.683 2007-08-01 12:45:56.683 9369 1 1540127 1179693 2007-07-19 15:40:46.300 2007-07-19 15:40:46.300 37376 1 9372141 9404884 14 2009-02-04 13:47:04.580 2009-03-09 10:37:15.007 8913230 9372140 9404883 1 2009-03-05 15:48:36.407 2009-03-05 15:48:36.407 9372076 1540128 1179693 2007-07-30 14:22:26.430 2007-07-30 14:22:26.430 6262 1 31555586 31588354 7 2009-03-05 15:48:36.407 2012-12-17 13:38:50.007 9372076 1540131 1179693 2007-08-01 12:12:37.257 2007-08-01 12:12:37.257 10063 1 9372145 9404887 9 2009-03-03 08:49:09.297 2009-03-09 08:24:56.687 9371999 10912054 10944813 24 2009-02-04 13:47:04.580 2009-11-11 09:39:18.543 8913230 2785317 2818082 28 2007-06-08 14:58:06.293 2007-12-18 11:49:30.567 1179693 31555585 31588353 114 2007-06-08 14:58:06.293 2014-10-22 14:24:49.050 1179693 1540130 1179693 2007-08-01 12:12:28.667 2007-08-01 12:12:28.667 11025 1 1540133 1179693 2007-08-01 12:12:53.703 2007-08-01 12:12:53.703 5576 1 1540132 1179693 2007-08-01 12:12:48.457 2007-08-01 12:12:48.457 4527 1 1540150 1179693 2007-08-01 15:53:00.820 2007-08-01 15:53:00.820 11522 1 9372166 9404907 13 2009-03-05 15:41:25.080 2009-03-09 16:50:13.197 9372068 1540151 1179693 2007-08-01 15:53:08.040 2007-08-01 15:53:08.040 10747 1 1540152 1179693 2007-08-01 15:53:16.873 2007-08-01 15:53:16.873 11758 1 1540153 1179693 2007-08-01 15:53:24.727 2007-08-01 15:53:24.727 10482 1 1540154 1179693 2007-08-01 15:53:31.020 2007-08-01 15:53:31.020 11087 1 9372170 9404911 16 2009-02-04 13:47:04.580 2009-03-10 09:05:05.047 8913230 1540155 1179693 2007-08-01 15:53:39.503 2007-08-01 15:53:39.503 11398 1 1540156 1179693 2007-08-01 15:53:46.357 2007-08-01 15:53:46.357 11159 1 9372168 9404909 15 2009-02-04 13:47:04.580 2009-03-09 12:31:59.233 8913230 1540157 1179693 2007-08-01 15:53:56.763 2007-08-01 15:53:56.763 10416 1 34406564 1409026 2026-02-04 08:21:44.129 2026-02-04 08:21:44.129 1540142 1179693 2007-08-01 12:46:45.853 2007-08-01 12:46:45.853 10160 1 34406563 1409026 2026-02-04 08:21:44.129 2026-02-04 08:21:44.129 1540143 1179693 2007-08-01 12:46:52.830 2007-08-01 12:46:52.830 10044 1 34406562 1409026 2026-02-04 08:21:44.129 2026-02-04 08:21:44.129 1540144 1179693 2007-08-01 12:47:01.187 2007-08-01 12:47:01.187 9288 1 34406561 1409026 2026-02-04 08:21:44.129 2026-02-04 08:21:44.129 1540145 1179693 2007-08-01 12:50:46.817 2007-08-01 12:50:46.817 666 1 34406560 1409026 2026-02-04 08:21:44.128 2026-02-04 08:21:44.128 1540146 1179693 2007-08-01 12:50:57.210 2007-08-01 12:50:57.210 2538 1 9372162 9404903 12 2009-03-05 15:41:25.080 2009-03-09 16:49:00.120 9372068 34406559 1409026 2026-02-04 08:21:44.128 2026-02-04 08:21:44.128 1540147 1179693 2007-08-01 12:51:09.677 2007-08-01 12:51:09.677 1359 1 34406558 1409026 2026-02-04 08:21:44.128 2026-02-04 08:21:44.128 5374467 5407203 2 2008-06-25 12:52:44.010 2008-06-25 12:55:18.017 5374441 1540148 1179693 2007-08-01 15:52:42.267 2007-08-01 15:52:42.267 12510 1 9372160 9404901 11 2009-03-05 15:41:25.080 2009-03-09 12:14:26.820 9372068 34406557 1409026 2026-02-04 08:21:25.809 2026-02-04 08:21:25.809 1540149 1179693 2007-08-01 15:52:55.127 2007-08-01 15:52:55.127 10069 1 5374480 5407216 3 2008-06-25 12:52:44.010 2008-06-25 15:16:13.890 5374441 9372179 9404920 16 2009-03-05 15:41:25.080 2009-03-10 09:27:24.053 9372068 9372182 9404923 17 2009-03-05 15:41:25.080 2009-03-10 10:31:45.753 9372068 5374481 5407217 3 2008-06-25 12:52:44.010 2008-06-25 15:16:13.890 5374441 9372184 9404925 18 2009-03-05 15:41:25.080 2009-03-10 10:52:29.357 9372068 19300409 1409026 2012-04-17 14:55:11.257 2012-04-17 14:55:11.257 9372185 9404926 19 2009-03-05 15:41:25.080 2009-03-10 10:56:41.610 9372068 21365482 21365241 21398204 21433961 2 2013-07-12 22:16:53.933 2013-07-16 12:04:09.063 21365487 9372172 9404913 14 2009-03-05 15:41:25.080 2009-03-09 17:06:53.617 9372068 1867849 1835085 5374487 5407223 6 2008-06-25 12:52:44.010 2008-06-25 15:46:27.643 5374441 19041452 19074138 31 2009-02-04 13:47:04.580 2011-03-21 12:59:38.190 8913230 5374490 5407226 8 2008-06-25 12:52:44.010 2008-06-25 16:04:38.360 5374441 19041454 19074140 29 2009-02-04 09:06:26.687 2011-04-15 08:30:45.777 8913216 5374489 5407225 7 2008-06-25 12:52:44.010 2008-06-25 15:57:05.487 5374441 9372175 9404916 15 2009-03-05 15:41:25.080 2009-03-10 09:11:38.777 9372068 21365488 21365241 21398210 21433991 3 2013-07-16 12:05:55.177 2013-07-17 16:21:56.080 21365489 21365522 19041456 19074142 10 2009-03-03 08:49:09.297 2009-03-09 14:48:30.467 9371999 5374483 5407219 4 2008-06-25 12:52:44.010 2008-06-25 15:43:48.657 5374441 1867843 1835079 1540099 1179693 2007-06-08 16:59:24.040 2007-06-08 16:59:24.040 121007 1 5374485 5407221 5 2008-06-25 12:52:44.010 2008-06-25 15:44:30.290 5374441 9372177 9404918 17 2009-02-04 13:47:04.580 2009-03-10 09:09:16.130 8913230 1540098 1179693 2007-06-08 16:59:05.020 2007-06-08 16:59:05.020 1773776 1 5374491 5407227 9 2008-06-25 12:52:44.010 2008-06-25 16:05:15.967 5374441 9372187 9404928 20 2009-03-05 15:41:25.080 2009-03-10 11:01:12.040 9372068 31555597 31588365 19 2009-03-12 15:26:12.667 2017-06-07 17:58:35.800 9372235 31555599 31588367 20 2009-03-12 15:26:12.667 2019-04-04 14:54:54.187 9372235 5374499 5407234 10 2008-06-25 12:52:44.010 2008-06-25 16:05:35.447 5374441 19234818 19202049 1179693 2012-03-19 15:39:36.730 2012-03-19 15:39:36.730 31555600 31588368 21 2009-03-12 15:26:12.667 2019-04-04 14:57:28.690 9372235 19234817 19202049 8913230 2012-03-19 15:39:14.610 2012-03-19 15:39:14.610 9372217 9404954 23 2009-03-05 15:41:25.080 2009-03-11 15:47:25.200 9372068 9404973 cd /mbari/Tempbox/mccann <103 elvis.shore.mbari.org /Tempbox/mccann> tar cvf 2012.268.07.tar 2012.268.07 2012.268.07/ 2012.268.07/SmartSamplerDataLog.log 2012.268.07/Gulper.log ... 2012.268.07/ctdDriver2.cfg 2012.268.07/gps.log 2012.268.07/m3dmgx1.log <105 elvis.shore.mbari.org /Tempbox/mccann> gzip 2012.268.07.tar {noformat} # Copy tarball to /usr/local/auv_ctd and run the auvportal which submits the data to SSDS: {noformat} <255 elvis.shore.mbari.org /Tempbox/mccann> cp 2012.268.07.tar.gz /usr/local/auv_ctd <256 elvis.shore.mbari.org /Tempbox/mccann> cd /usr/local/auv_ctd <257 elvis.shore.mbari.org /local/auv_ctd> ./portal AUVCTD2012268 2012.268.07.tar.gz Tue, 25 Sep 2012 09:27:43 PDT: processConnection() got message: AUVCTD2012268;2012.268.07.tar.gz dstDir.getName() = 2012268 Unpacking /usr/local/auv_ctd/2012.268.07.tar.gz Copying /usr/local/auv_ctd/2012268/2012.268.07/ctdDriver2.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/ctdDriver2.log Copying /usr/local/auv_ctd/2012268/2012.268.07/gps.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/gps.log Copying /usr/local/auv_ctd/2012268/2012.268.07/isuslog.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/isuslog.log Copying /usr/local/auv_ctd/2012268/2012.268.07/devices.cfg to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/devices.cfg ... DEBUG [Thread-0] (XmlBuilder.java:538) - appendPrimitiveType called with method name isValidRole on object of class moos.ssds.metadata.DataProducer DEBUG [Thread-0] (XmlBuilder.java:482) - appendElement called on class moos.ssds.metadata.DataProducer using method getClass DEBUG [Thread-0] (XmlBuilder.java:278) - Converting java.lang.Class(class moos.ssds.metadata.DataProducer) to XML Marshalled the AUV mission in /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 into XML Attempting to publish AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ published XML describing mission AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ If everything goes well in SSDS:Ingest/Ruminate an email message to auvctd@listserver.mbari.org will arrive shortly. Tue, 25 Sep 2012 09:34:29 PDT: Done receiving AUV mission data. {noformat} Note: With the move of the portal to from jeckel to elvis the mailing to auvctd@listserver.mbari.org has stopped working. This needs to be debugged and fixed. # Process the LOPC data, e.g.: {noformat} <109 elvis.shore.mbari.org /u/ssdsadmin> cd ~/dev/auv_ctd/src/py/auv/ <109 elvis.shore.mbari.org /u/ssdsadmin> /bin/bash <109 elvis.shore.mbari.org /u/ssdsadmin> export PATH="/usr/local/miniconda2/bin:$PATH" <109 elvis.shore.mbari.org /u/ssdsadmin> source activate auv_ctd (auv_ctd) bash-4.2$ processLOPC.pl 2013.224.02 (auv_ctd) bash-4.2$ exit {noformat} # Once the mission's instrument netcdf files have been created (check pismo:/opt/ssds/logs/ssds-updateBot.log for successful processing) the overall Dorado science data processing script can be edited with the new survey to be processed: {noformat} <102 elvis.shore.mbari.org /u/ssdsadmin> cd ~/dev/auv_ctd/src/matlab/doradosdp/process/ <103 elvis.shore.mbari.org /doradosdp/process> vi reprocess_surveys.m {noformat} # Add a line for the new survey to the end of the reprocess_surveys.m file, commenting out the previous survey that was processed: {noformat} % CANON September 2012 - laps around ESP drifter - last one for this campaign %%AUVprocess_main('AUVCTD', 2012, ['258.00'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') % CANON September 2012 - laps around ESP drifter - last one for this campaign AUVprocess_main('AUVCTD', 2012, ['268.07'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') {noformat} # Execute the Matlab job with a shell script that properly sets the DISPLAY environment and executes reprocess_surveys.m with matlab: {noformat} <104 elvis.shore.mbari.org /doradosdp/process> RunReprocessJob {noformat} # Check for error messages. If everything works an email is sent to the notifyEmail setting in reprocess_surveys.m, usually auvctd@listserver.mbari.org. h2. Software checkout and repository code management The Software for both the AUV Portal and the Science data processing are located in the _auv_ctd_ module on the CVS server moonjelly.shore.mbari.org. To check the software to a system use your favorite CVS client, or on Unix use these commands: {noformat} cd cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). {noformat} Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): {noformat} {noformat} > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... {noformat} h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: {noformat} usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy {noformat} For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: {noformat} adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 {noformat} Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look something like this: {noformat} bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ {noformat} \\ On the portal computer the screen will display text something like this: {noformat} Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS {noformat} Note: Detailed command-line instructions for "portaling" AUVCTD mission data into SSDS are available in the README file in the project or in the operational location on jeckel.shore.mbari.org in /usr/local/auv_ctd/ directory. There is also the portal.log file there which logs the output of the deamon process running on jeckel that receives and processes the data from AUV uploads from the vehicle. The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: {noformat} > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> {noformat} So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: {noformat} Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds {noformat} +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables and an Ocean Data View spreadsheet file. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file). If one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files.]]> 9372235 9372207 9404946 22 2009-03-05 15:41:25.080 2009-03-11 14:15:24.467 9372068 9372205 9404944 21 2009-03-05 15:41:25.080 2009-03-10 11:22:37.413 9372068 19041450 19074136 6 2009-03-05 15:48:36.407 2010-08-24 16:34:22.543 9372076 19041448 19074134 2 2011-05-23 10:13:56.883 2012-12-17 13:33:47.933 16777434 5374524 5407259 11 2008-06-25 12:52:44.010 2008-06-25 16:20:43.927 5374441 19041447 19074133 1 2011-05-23 10:13:56.883 2011-05-23 10:13:56.883 16777434 29819155 29851905 18 2009-03-12 15:26:12.667 2013-09-17 08:18:51.077 9372235 16810195 16777434 27656197 8913230 2015-07-06 08:21:41.060 2015-07-06 08:21:41.060 21528613 21365241 2013-06-27 08:16:15.687 2013-06-27 08:16:15.687 493397 1 27656196 8913230 2015-07-06 08:21:41.060 2015-07-06 08:21:41.060 9372239 9404977 2 2009-03-12 15:26:12.667 2009-03-12 15:28:56.557 9372235 9372241 9404978 81 2007-06-08 14:58:06.293 2009-03-09 12:07:08.927 1179693 21528614 21365241 2013-06-27 08:16:15.753 2013-06-27 08:16:15.753 283869 1 1540160 1179693 2007-08-01 15:58:51.813 2007-08-01 15:58:51.813 10910 1 1540159 1179693 2007-08-01 15:54:07.333 2007-08-01 15:54:07.333 10943 1 1540158 1540158 1179693 2007-08-01 15:54:07.333 2007-08-01 15:54:15.760 10943 2 9372237 9404975 1 2009-03-12 15:26:12.667 2009-03-12 15:26:12.667 9372235 21528624 21365241 2013-08-05 16:22:55.613 2013-08-05 16:22:55.613 170751 1 21365395 21365241 21398119 3 2013-07-11 11:17:55.807 2013-07-11 13:47:35.097 21365401 21365410 10355514 10388244 68 2009-03-05 15:41:25.080 2009-07-16 17:00:14.307 9372068 27656193 8913230 2015-07-06 08:21:40.970 2015-07-06 08:21:40.970 27656194 8913230 2015-07-06 08:21:41.060 2015-07-06 08:21:41.060 10912037 10944796 21 2009-02-04 13:47:04.580 2009-06-22 13:50:41.383 8913230 27656195 8913230 2015-07-06 08:21:41.060 2015-07-06 08:21:41.060 21528636 8913162 2009-02-02 16:30:52.167 2013-08-12 13:34:32.607 8516 7 9076758 10912039 10944798 22 2009-02-04 13:47:04.580 2009-11-11 08:40:51.207 8913230 21528634 8913162 2009-02-02 16:30:52.167 2009-03-12 15:03:21.540 8621 5 9076758 21528635 8913162 2009-02-02 16:30:52.167 2013-08-12 13:16:58.587 8620 6 9076758 21433413 20971983 2013-07-03 13:40:54.490 2013-07-03 13:40:54.490 5439534 1179693 2008-07-08 11:36:45.667 2008-07-08 11:36:45.667 1716736 1 21433412 20971983 2013-07-03 13:40:54.490 2013-07-03 13:40:54.490 21004746 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 h4. 16 May 2013 Another post to the pgsql-perform mail list asking for opinions on a more conventional server configuration: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 3:36:03 PM PDT To: pgsql-performance@postgresql.org Cc: Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On May 7, 2013, at 4:21 PM, Jeff Janes wrote: On Thu, May 2, 2013 at 6:35 PM, Scott Marlowe wrote: On Thu, May 2, 2013 at 5:11 PM, Mike McCann wrote: > Hello, > > We are in the fortunate situation of having more money than time to help > solve our PostgreSQL 9.1 performance problem. > > Our server hosts databases that are about 1 GB in size with the largest > tables having order 10 million 20-byte indexed records. The data are loaded > once and then read from a web app and other client programs. Some of the > queries execute ORDER BY on the results. There are typically less than a > dozen read-only concurrent connections to any one database. I wouldn't count on this being a problem that can be fixed merely by throwing money at it. How many rows does any one of these queries need to access and then ORDER BY? ... > > HP ProLiant DL360p Gen 8 > Dual Intel Xeon 2.4GHz 4-core E5-2609 CPUs > 64GB RAM > 2x146GB 15K SAS hard drives > 3x200GB SATA SLC SSDs > + the usual accessories (optical drive, rail kit, dual power supplies) If your DB is 1G, and will grow to 10G then the IO shouldn't be any problem, as the whole db should be cached in memory. But it can take a surprisingly long time to get it cached in the first place, from a cold start. If that is the problem, pg_prewarm could help. Cheers, Jeff Thank you everyone for your suggestions. It's clear that our current read performance was not limited by hardware. An 'explain analyze' for a sample query is: stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=541002.15..549456.68 rows=3381814 width=20) (actual time=6254.780..7244.074 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..354.385 rows=3381814 loops=1) Total runtime: 7425.854 ms (5 rows) Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? We have an alternate (a bit more conventional) server configuration that we are considering: HP ProLiant DL360p Gen 8 Dual Intel Xeon 3.3GHz 4-core E5-2643 CPUs 128GB PC3-12800 RAM 16x146GB 15K SAS hard drives HP Smart Array P822/2GB FBWC controller + P420i w/ 2GB FBWC + the usual accessories (optical drive, rail kit, dual power supplies) All suggestions welcomed! -Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} The responses: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 4:58:17 PM PDT To: Jeff Janes Cc: "pgsql-performance@postgresql.org" , Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden On May 13, 2013, at 4:24 PM, Jeff Janes wrote: > On Mon, May 13, 2013 at 3:36 PM, Mike McCann wrote: > > Increasing work_mem to 355 MB improves the performance by a factor of 2: > > stoqs_march2013_s=# set work_mem='355MB'; > SET > stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > > I tried changing random_page_cost to from 4 to 1 and saw no change. > > I'm wondering now what changes might get this query to run in less than one second. I think you are worrying about the wrong thing here. What is a web app going to do with 3,381,814 rows, once it obtains them? Your current testing is not testing the time it takes to stream that data to the client, or for the client to do something meaningful with that data. If you only plan to actually fetch a few dozen of those rows, then you probably need to incorporate that into your test, either by using a LIMIT, or by using a mock-up of the actual application to do some timings. Also, what is the type and collation of the column you are sorting on? non-'C' collations of text columns sort about 3 times slower than 'C' collation does. If all the data is in memory, then will faster CPU and memory be the things that help? Yes, those would help (it is not clear to me which of the two would help more), but I think you need to rethink your design of sending the entire database table to the application server for each page-view. Cheers, Jeff Hi Jeff, The datavalue column is double precision: stoqs_march2013_s=# \d+ stoqs_measuredparameter Table "public.stoqs_measuredparameter" Column | Type | Modifiers | Storage | Description ----------------+------------------+----------------------------------------------------------------------+---------+------------- id | integer | not null default nextval('stoqs_measuredparameter_id_seq'::regclass) | plain | measurement_id | integer | not null | plain | parameter_id | integer | not null | plain | datavalue | double precision | not null | plain | Indexes: "stoqs_measuredparameter_pkey" PRIMARY KEY, btree (id) "stoqs_measuredparameter_measurement_id_parameter_id_key" UNIQUE CONSTRAINT, btree (measurement_id, parameter_id) "stoqs_measuredparameter_datavalue" btree (datavalue) "stoqs_measuredparameter_measurement_id" btree (measurement_id) "stoqs_measuredparameter_parameter_id" btree (parameter_id) Foreign-key constraints: "stoqs_measuredparameter_measurement_id_fkey" FOREIGN KEY (measurement_id) REFERENCES stoqs_measurement(id) DEFERRABLE INITIALLY DEFERRED "stoqs_measuredparameter_parameter_id_fkey" FOREIGN KEY (parameter_id) REFERENCES stoqs_parameter(id) DEFERRABLE INITIALLY DEFERRED Has OIDs: no Thanks for the suggestion and advice to examine the web app performance. We've actually taken quite a few steps to optimize how the web app works. The example query I provided is a simple worst-case one that we can use to help us decide on the proper hardware. An actual query performed by the web app is: stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_march2013_s-# stoqs_parameter.name AS parameter__name, stoqs_march2013_s-# stoqs_parameter.standard_name AS parameter__standard_name, stoqs_march2013_s-# stoqs_measurement.depth AS measurement__depth, stoqs_march2013_s-# stoqs_measurement.geom AS measurement__geom, stoqs_march2013_s-# stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_march2013_s-# stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_march2013_s-# stoqs_measuredparameter.datavalue AS datavalue, stoqs_march2013_s-# stoqs_parameter.units AS parameter__units stoqs_march2013_s-# FROM stoqs_parameter p1, stoqs_march2013_s-# stoqs_measuredparameter stoqs_march2013_s-# INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) stoqs_march2013_s-# INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) stoqs_march2013_s-# INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) stoqs_march2013_s-# INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) stoqs_march2013_s-# INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) stoqs_march2013_s-# INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id stoqs_march2013_s-# WHERE (p1.name = 'sea_water_sigma_t') stoqs_march2013_s-# AND (mp1.datavalue > 25.19) stoqs_march2013_s-# AND (mp1.datavalue < 26.01) stoqs_march2013_s-# AND (mp1.parameter_id = p1.id) stoqs_march2013_s-# AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' stoqs_march2013_s(# AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' stoqs_march2013_s(# AND stoqs_parameter.name IN ('fl700_uncorr') stoqs_march2013_s(# AND stoqs_measurement.depth >= -1.88 stoqs_march2013_s(# AND stoqs_platform.name IN ('dorado') stoqs_march2013_s(# AND stoqs_measurement.depth <= 83.57) stoqs_march2013_s-# ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=770.211..770.211 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=770.200..770.200 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=770.036..770.036 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=91.369..680.072 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=91.346..577.633 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=91.280..531.408 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=91.202..116.140 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.002..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=91.194..109.846 rows=20170 loop s=1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=90.794..90.794 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=62.769..62.769 rows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=27.412..2 7.412 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.008..0. 017 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.012..0.012 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.007..0.010 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.004..0.004 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.004..0.004 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.151..0.151 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.035..0.145 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.005..0.069 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.014..0.014 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.008..0.012 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 770.445 ms (42 rows) We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Thanks, Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} Another response: {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 6:09:58 PM PDT To: Mike McCann Cc: Jeff Janes , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden On Mon, May 13, 2013 at 5:58 PM, Mike McCann wrote: > We assume that steps taken to improve the worst-case query scenario will > also improve these kind of queries. If anything above pops out as needing > better planning please let us know that too! Bad assumption. If your real workload will be queries like the one here that takes 700 ms, but you'll be running 10,000 of them a second, you're tuning / hardware choices are going to be much different then if your query is going to be the previous 7 second one. Use realistic queries, not ones that are nothing like what your real ones will be. then use pgbench and its ability to run custom sql scripts to get a REAL idea how your hardware performs. Note that if you will run the slow query you posted like once a minute and roll it up or cache it then don't get too worried about it. Pay attention to the queries that will add up, in aggregate, to your greatest load. {noformat} Not a whole lot of opinions except that tuning a server requires diligent work and performance is something one must do some work to achieve, and cannot be had by simply throwing money at the problem. h4. 20 May 2013 A few more responses over the weekend: {noformat} From: Greg Smith Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 7:44:16 PM PDT To: Mike McCann Cc: pgsql-performance@postgresql.org, Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On 5/13/13 6:36 PM, Mike McCann wrote: > stoqs_march2013_s=# explain analyze select * from > stoqs_measuredparameter order by datavalue; > > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual > time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 > rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > I tried changing random_page_cost to from 4 to 1 and saw no change. Have you tried putting an index by datavalue on this table? Once you've done that, then changing random_page_cost will make using that index look less expensive. Sorting chews through a good bit of CPU time, and that's where all of your runtime is being spent at--once you increase work_mem up very high that is. > I'm wondering now what changes might get this query to run in less than > one second. If all the data is in memory, then will faster CPU and > memory be the things that help? You're trying to fix a fundamental design issue with hardware. That usually doesn't go well. Once you get a box big enough to hold the whole database in RAM, beyond that the differences between server systems are relatively small. -- Greg Smith 2ndQuadrant US greg@2ndQuadrant.com Baltimore, MD PostgreSQL Training, Services, and 24x7 Support www.2ndQuadrant.com {noformat} and {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 8:57:32 PM PDT To: Greg Smith Cc: Mike McCann , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden , Jeff Janes On Sun, May 19, 2013 at 8:44 PM, Greg Smith wrote: >On 5/13/13 6:36 PM, Mike McCann wrote: >> >> stoqs_march2013_s=# explain analyze select * from >> stoqs_measuredparameter order by datavalue; >> >> QUERY PLAN >> >>------------------------------------------------------------------------------------------------------------------------------------------ >> Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual >> time=2503.078..2937.130 rows=3381814 loops=1) >> Sort Key: datavalue >> Sort Method: quicksort Memory: 362509kB >> -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 >> rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) >> Total runtime: 3094.601 ms >> (5 rows) >> >> I tried changing random_page_cost to from 4 to 1 and saw no change. > > > Have you tried putting an index by datavalue on this table? Once you've > done that, then changing random_page_cost will make using that index look > less expensive. Sorting chews through a good bit of CPU time, and that's > where all of your runtime is being spent at--once you increase work_mem up > very high that is. This++ plus cluster on that index if you can. {noformat} h4. 2 July 2013 We made the decision made to purchase a server with a FusionIO card. Server is now set up on the network with name "kraken". Here is the [confluence page for kraken's configuration settings| PostgreSQL Server kraken setup]. Setting up ability to run pgbench-tools on all the various servers. This may be done per user account by first installing from git and following the instructions in README.rst; however, the scripts need to run commands requiring createdb and vacuum privileges, so it's best to run as user postgres. So, logging into pgtest as user postgres and getting things set up: {noformat} -bash-4.1$ pwd /var/lib/pgsql -bash-4.1$ mkdir dev -bash-4.1$ cd dev -bash-4.1$ git clone git://git.postgresql.org/git/pgbench-tools.git Initialized empty Git repository in /var/lib/pgsql/dev/pgbench-tools/.git/ remote: Counting objects: 108, done. remote: Compressing objects: 100% (106/106), done. remote: Total 108 (delta 54), reused 0 (delta 0) Receiving objects: 100% (108/108), 24.61 KiB, done. Resolving deltas: 100% (54/54), done. -bash-4.1$ createdb results -bash-4.1$ createdb pgbench -bash-4.1$ cd pgbench-tools/ -bash-4.1$ psql -f init/resultdb.sql -d results psql:init/resultdb.sql:1: ERROR: table "timing" does not exist CREATE TABLE CREATE INDEX psql:init/resultdb.sql:12: ERROR: table "tests" does not exist psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_test_seq" for serial column "tests.test" psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_set_seq" for serial column "tests.set" CREATE TABLE psql:init/resultdb.sql:31: ERROR: table "testset" does not exist psql:init/resultdb.sql:36: NOTICE: CREATE TABLE will create implicit sequence "testset_set_seq" for serial column "testset.set" CREATE TABLE INSERT 0 1 psql:init/resultdb.sql:40: ERROR: table "test_bgwriter" does not exist CREATE TABLE -bash-4.1$ psql -c "UPDATE testset SET info='Initial test set' WHERE set=1" -d results UPDATE 1 {noformat} I think those ERROR messages can be ignored because the following tests work. The gnuplot package is needed for producing graphs of the results: {noformat} sudo -c "yum install gnuplot" {noformat} Edit the config file with hard path to pgbench and reduced number of scale and client runs so that we have a quick execution to just test things: {noformat} -bash-4.1$ mv config config.130702 -bash-4.1$ cp config.130702 config -bash-4.1$ vi config -bash-4.1$ diff config.130702 config 8c8 < PGBENCHBIN=`which pgbench` --- > PGBENCHBIN=/usr/pgsql-9.1/bin/pgbench 42,44c42,47 < SCALES="1 10 100 1000" < SETCLIENTS="1 2 4 8 16 32" < SETTIMES=3 --- > SCALES="1" > SETCLIENTS="1" > SETTIMES=1 > #SCALES="1 10 100 1000" > #SETCLIENTS="1 2 4 8 16 32" > #SETTIMES=3 {noformat} Edit scripts to use "local socket" connection (removing "-h $TESTHOST") for the postgres user: {noformat} -bash-4.1$ diff runset.130702 runset 19c19 < TESTPSQL="psql -h $TESTHOST -U $TESTUSER -p $TESTPORT -d $TESTDB" --- > TESTPSQL="psql -U $TESTUSER -p $TESTPORT -d $TESTDB" 24c24 < $PGBENCHBIN -i -s $SCALE -h $TESTHOST -U $TESTUSER -p $TESTPORT $TESTDB --- > $PGBENCHBIN -i -s $SCALE -U $TESTUSER -p $TESTPORT $TESTDB -bash-4.1$ diff benchwarmer.130702 benchwarmer 84,85c84,85 < TESTPSQL="psql -h $TESTHOST -U $TESTUSER -p $TESTPORT -d $TESTDB" < RESULTPSQL="psql -h $RESULTHOST -U $RESULTUSER -p $RESULTPORT -d $RESULTDB" --- > TESTPSQL="psql -U $TESTUSER -p $TESTPORT -d $TESTDB" > RESULTPSQL="psql -U $RESULTUSER -p $RESULTPORT -d $RESULTDB" 158c158 < $PGBENCHBIN -f $BASEDIR/$TESTDIR/$SCRIPT -s $SCALE -l -n $TESTLEN -U $TESTUSER -h $TESTHOST -p $TESTPORT -c $CLIENTS $WORKERS $TESTDB > results.txt & --- > $PGBENCHBIN -f $BASEDIR/$TESTDIR/$SCRIPT -s $SCALE -l -n $TESTLEN -U $TESTUSER -p $TESTPORT -c $CLIENTS $WORKERS $TESTDB > results.txt & -bash-4.1$ diff webreport.130702 webreport 10c10 < RESULTPSQL="psql -h $RESULTHOST -U $RESULTUSER -p $RESULTPORT -d $RESULTDB" --- > RESULTPSQL="psql -U $RESULTUSER -p $RESULTPORT -d $RESULTDB" {noformat} And now a test run looks like: {noformat} -bash-4.1$ ./runset Removing old pgbench tables DROP TABLE VACUUM Creating new pgbench tables NOTICE: table "pgbench_branches" does not exist, skipping NOTICE: table "pgbench_tellers" does not exist, skipping NOTICE: table "pgbench_accounts" does not exist, skipping NOTICE: table "pgbench_history" does not exist, skipping creating tables... 10000 tuples done. 20000 tuples done. 30000 tuples done. 40000 tuples done. 50000 tuples done. 60000 tuples done. 70000 tuples done. 80000 tuples done. 90000 tuples done. 100000 tuples done. set primary key... NOTICE: ALTER TABLE / ADD PRIMARY KEY will create implicit index "pgbench_branches_pkey" for table "pgbench_branches" NOTICE: ALTER TABLE / ADD PRIMARY KEY will create implicit index "pgbench_tellers_pkey" for table "pgbench_tellers" NOTICE: ALTER TABLE / ADD PRIMARY KEY will create implicit index "pgbench_accounts_pkey" for table "pgbench_accounts" vacuum...done. Run set #1 of 1 with 1 clients scale=1 Running tests using: psql -U postgres -p 5432 -d pgbench Storing results using: psql -U postgres -p 5432 -d results Cleaning up database pgbench TRUNCATE TABLE VACUUM CHECKPOINT INSERT 0 1 This is test 3 Script select.sql executing for 1 concurrent users... UPDATE 1 transaction type: Custom query scaling factor: 1 query mode: simple number of clients: 1 number of threads: 1 duration: 60 s number of transactions actually processed: 547599 tps = 9126.622772 (including connections establishing) tps = 9127.071214 (excluding connections establishing) Worst latency results: 2018 2311 2370 3155 3476 Could not find/open font when opening font "arial", using internal non-scalable font Could not find/open font when opening font "arial", using internal non-scalable font Could not find/open font when opening font "arial", using internal non-scalable font Warning: empty x range [1:1], adjusting to [0.99:1.01] Warning: empty y range [9228:9228], adjusting to [9135.72:9320.28] Warning: empty y2 range [21:21], adjusting to [20.79:21.21] Could not find/open font when opening font "arial", using internal non-scalable font Warning: empty x range [1:1], adjusting to [0.99:1.01] Warning: empty y range [9228:9228], adjusting to [9135.72:9320.28] Could not find/open font when opening font "arial", using internal non-scalable font Warning: empty x range [1:1], adjusting to [0.99:1.01] Warning: empty y range [1:1], adjusting to [0.99:1.01] Warning: empty z range [9228:9228], adjusting to [9135.72:9320.28] {noformat} h4. 3 July 2013 Making the results visible on an internally accessible web page. Turn on httpd and make an alias to the results directory that is produced above: {noformat} sudo /bin/bash chkconfig httpd on /sbin/service httpd start {noformat} Confirm that http://pgtest.shore.mbari.org/ is visible. It wasn't because the firewall on pgtest was preventing it. A call to Pat Allen fixed this. Next, edit /etc/httpd/conf/httpd.conf to add the alias and make the postgres home dir world executable (don't do this on a production system!) so that the apache httpd process can serve the directory: {noformat} # diff httpd.conf.130703 httpd.conf 559a560,569 > # Expose our pgbench-toolbox results > Alias /pgbench /var/lib/pgsql/dev/pgbench-tools/results > > > Options Indexes FollowSymLinks > AllowOverride None > Order allow,deny > Allow from all > > # chmod +x /var/lib/pgsql {noformat} The pgbench-toolbox results can now be shared at http://pgtest.shore.mbari.org/pgbench/ ]]> 20971983 9798432 9831183 42 2009-03-05 15:41:25.080 2009-05-06 14:09:05.593 9372068 19041640 19074320 108 2007-06-08 14:58:06.293 2013-01-10 16:22:56.447 1179693 19041612 19074293 104 2007-06-08 14:58:06.293 2013-01-10 16:12:23.800 1179693 19041614 19074295 105 2007-06-08 14:58:06.293 2013-01-10 16:17:59.613 1179693 29819111 29851879 40 2009-02-04 09:06:26.687 2016-12-20 07:43:15.853 8913216 8913329 8946078 53 2007-06-08 14:58:06.293 2009-02-06 17:58:59.617 1179693 19041615 19074296 106 2007-06-08 14:58:06.293 2013-01-10 16:19:53.193 1179693 19041617 19074298 107 2007-06-08 14:58:06.293 2013-01-10 16:20:13.707 1179693 8913328 8946077 52 2007-06-08 14:58:06.293 2009-02-05 16:41:19.063 1179693 21433404 20971983 2013-07-03 13:40:54.490 2013-07-03 13:40:54.490 10355368 10388099 63 2009-03-05 15:41:25.080 2009-07-07 13:45:27.257 9372068 21433405 20971983 2013-07-03 13:40:54.490 2013-07-03 13:40:54.490 3604643 17 1179693 2008-05-06 10:27:15.047 2013-02-04 14:43:15.030 21433406 20971983 2013-07-03 13:40:54.490 2013-07-03 13:40:54.490 8913331 8946080 54 2007-06-08 14:58:06.293 2009-02-06 18:03:02.117 1179693 10355370 10388101 64 2009-03-05 15:41:25.080 2009-07-13 16:17:48.757 9372068 21433407 20971983 2013-07-03 13:40:54.490 2013-07-03 13:40:54.490 5439528 1179693 2008-06-30 15:47:56.740 2008-06-30 15:47:56.740 28160 1 21433408 20971983 2013-07-03 13:40:54.490 2013-07-03 13:40:54.490 21433409 20971983 2013-07-03 13:40:54.490 2013-07-03 13:40:54.490 21433410 20971983 2013-07-03 13:40:54.490 2013-07-03 13:40:54.490 16777416 16810178 1 2009-02-23 08:37:11.143 2009-02-23 08:37:11.143 9371817 21433411 20971983 2013-07-03 13:40:54.490 2013-07-03 13:40:54.490 19041597 19074278 99 2007-06-08 14:58:06.293 2011-07-21 10:53:35.430 1179693 19041606 19074287 100 2007-06-08 14:58:06.293 2013-01-09 14:38:46.500 1179693 16777429 16810191 94 2007-06-08 14:58:06.293 2010-03-15 16:11:16.883 1179693 19041609 19074290 102 2007-06-08 14:58:06.293 2013-01-10 16:09:55.633 1179693 16777431 16810193 95 2007-06-08 14:58:06.293 2011-05-23 10:08:12.920 1179693 19041610 19074291 103 2007-06-08 14:58:06.293 2013-01-10 16:11:53.203 1179693 16777432 16810194 96 2007-06-08 14:58:06.293 2011-05-23 10:12:38.387 1179693 19041607 19074288 101 2007-06-08 14:58:06.293 2013-01-10 14:50:03.723 1179693 16777435 16810196 97 2007-06-08 14:58:06.293 2011-05-23 10:13:38.940 1179693 26640961 26673717 113 2007-06-08 14:58:06.293 2014-09-09 07:57:35.497 1179693 2163070 2195834 27 2007-06-08 14:58:06.293 2007-12-18 11:48:48.973 1179693 2163069 2195833 26 2007-06-08 14:58:06.293 2007-12-18 11:27:45.450 1179693 2163068 2195832 25 2007-06-08 14:58:06.293 2007-10-03 11:57:53.670 1179693 1212691 [#frozen Pos] [excel]\n\n" unless -d $ARGV[0]; $showOver = $ARGV[1] || 60; # Show if pos repeats for more times than this $outputExcel = 0; $outputExcel = 1 if ( $ARGV[2] =~ /excel/i ); $| = 1; $deltaT = 5 / 60; # # Open directory and get list of all the raw nav files in it. Handle vnta & tibr # opendir (NAVDIR, $ARGV[0]) || die "Can't open directory $ARGV[0]: $!\n"; if ( $ARGV[0] =~ /vnta/ ) { @navfiles = grep { /vnta.txt$/ } readdir (NAVDIR); } elsif ( $ARGV[0] =~ /tibr/ ) { @navfiles = grep { /tibr.txt$/ } readdir (NAVDIR); } else { die "$0 works only for vnta or tibr directories.\n"; } die "Empty \@navFiles list. Was reading from: $ARGV[0]\n" unless @navfiles; closedir (NAVDIR); # # Print header for output # - the aveSD column is the average standard deviation between the original and edited nav # if the values are 0.00 then the original and edited data are identical. Larger # values will indicate a degraded performance in the nav system. There will natuarlly # be a correlation with performance accuracy and depth. # if ( $outputExcel ) { ##print "File,esecs,Lat,Lon,maxPres,numFreezes,totalDuration,recordDuration,aveSD,editedData\n"; print "File\tesecs\tLat\tLon\tmaxPres\tnumFreezes\ttotalDuration\trecordDuration\taveSD\teditedData\n"; } else { print "File str (hhmmss) end froz Lat froz lon sPres ePres # duration\n"; } # # Loop through all the files and collect stats on frozen positions # foreach ( @navfiles ) { $file = $_; print "file = $file\n" if $debug; if ( $outputExcel ) { } else { print "$file\n"; } calcFreezes(); calcSD(); } # End foreach #------------------------------------ # Subroutines: just to partition the work. No args passed. Use global vars # sub calcFreezes { open (NAVFILE, "$ARGV[0]/${file}") || die "Can't open file $ARGV[0]/${file}: $!\n"; $oldLat = 0; $oldLon = 0; $sameCount = 0; $notSame = 0; $prFile = $file; # Print file string, for listing below $prFile =~ s/vnta.txt//; $timeSum = 0; $recSum = 0; $numFrozen = 0; $maxPres = 0; while ( ) { next unless /^\d/; # Skip rec unless it begins with number # # Read record into my variables # ($Year, $Day, $Time, $Usec, $Lat, $Lon, $East, $North, $Pres, $Head, $Alti, $Pitch, $Roll, $PosFlag, $PresFlag, $HeadFlag, $AltiFlag, $AttitFlag) = split (',', $_); if ( $maxPres < $Pres ) { $maxPres = $Pres; } # # Count records of unchanging Lat & Lon # print "$_\n" if $debug; if ( $Lat == $oldLat && $Lon == $oldLon ) { $sameCount++; if ( $sameCount == 1 ) { $begUsec = $Usec; $begTime = $Time; $begPres = $Pres; } } elsif ( $oldLat != 0 ) { $notSame = 1; $timeDiffSec = $Usec - $begUsec; $duration = sprintf("%6.1f min", $timeDiffSec / 60); } # End if ( $Lat ... # # Print out stats for frozen position after we've counted them # if ( $sameCount > $showOver && $notSame ) { if ( $outputExcel ) { } else { printf ("%10s %d to %d %f %f %5.1f to %5.1F %3d %s\n", $prFile, $begTime, $Time, $oldLat, $oldLon, $begPres, $Pres, $sameCount, $duration); $prFile = ' '; } $recSum += $deltaT * $sameCount; $sameCount = 0; $notSame = 0; $timeSum += $duration; $numFrozen++; ##die "Debugging\n";; } # # Reset counter if not over our $showOver criteria # elsif ( $sameCount <= $showOver && $notSame ) { $sameCount = 0; $notSame = 0; } # End if ( $sameCount > $oldLat = $Lat; $oldLon = $Lon; } # End while (NAVFILE # # Print summary for this day # if ( $outputExcel ) { ##print "$prFile,$Usec,$Lat,$Lon,$maxPres,$numFrozen,$timeSum,", sprintf("%6.f", $recSum); print "$prFile\t$Usec\t$Lat\t$Lon\t$maxPres\t$numFrozen\t$timeSum\t", sprintf("%6.f", $recSum); } else { if ( $numFrozen > 0 ) { printf (" %3d frozen positions over %3d records long lasting total of %5.1f minutes\n", $numFrozen, $showOver, $timeSum); } } # End if ( $outputExcel } # End calcFreezes() sub calcSD { # # Create list of lat & lon of differences for the day # open (NAVFILE, "$ARGV[0]/${file}") || die "Can't open file $ARGV[0]/${file}: $!\n"; $editedFile = $file; $editedFile =~ s/\.txt/edited.txt/; if ( -f "$ARGV[0]/${editedFile}" ) { open (EDITEDNAV, "$ARGV[0]/${editedFile}") || die "Can't open file $ARGV[0]/${editedFile}: $!\n"; } else { print STDERR "Can't open file $ARGV[0]/${editedFile}: $!\n"; print STDERR "Skipping this file.\n"; if ( $outputExcel ) { ##print ",0,NoEditedNav\n"; print "\t0\tNoEditedNav\n"; } return; } undef @lonDiff; undef @latDiff; while ( ($raw = ) && ($edited = ) ) { next unless $raw =~ /^\d/; # Skip rec unless it begins with number # # Read raw record into my variables # my ($Year, $Day, $Time, $Usec, $Lat, $Lon, $East, $North, $Pres, $Head, $Alti, $Pitch, $Roll, $PosFlag, $PresFlag, $HeadFlag, $AltiFlag, $AttitFlag) = split (',', $raw); # # Read edited record into my variables # my (undef, undef, undef, $eUsec, $eLat, $eLon, undef, undef, $ePres, undef, undef, undef, undef, undef, undef, undef, undef, undef) = split (',', $edited); ##print "Lat diff = ", $Lat - $eLat, " Lon diff = ", $Lon - $eLon, "\n"; push @latDiff, $Lat - $eLat; push @lonDiff, $Lon - $eLon; } # End while ( # # Compute stats on pos differences # undef $lonStats; undef $latStats; my $lonStats = Statistics::Descriptive::Full->new(); my $latStats = Statistics::Descriptive::Full->new(); $lonStats->add_data(@lonDiff); $latStats->add_data(@latDiff); if ( $outputExcel ) { printf "\t %9.6f\n", ( sqrt($lonStats->variance) + sqrt($latStats->variance) ) / 2; } else { printf " lonDiff SD = %9.6f latDiff SD = %9.6f, Ave = %9.6f\n", sqrt($lonStats->variance), sqrt($latStats->variance), (sqrt($lonStats->variance) + sqrt($latStats->variance) ) / 2; } } # End calcSD() {noformat} The Perl script is executed with this shell script: {noformat}  #!/bin/sh # # Run stats on all of our ROV nav data outDir="/hosts/tempest/tempbox/mccann/navstats" for rov in tibr vnta do for yrdir in /hosts/tornado/TempNav/200[6-7] do dir="$yrdir/$rov" year=`echo $yrdir | cut -d/ -f5` echo navStats.pl $dir 60 excel ">" $outDir/navStats${year}${rov}.txt /bin/rm -f $outDir/navStats${year}${rov}.txt navStats.pl $dir 60 excel > $outDir/navStats${year}${rov}.txt done # # Concatenate all the ROV stat files into one # cat $outDir/navStats????${rov}.txt > $outDir/navStats${rov}.txt done {noformat}]]> 1179926 9502744 8913230 2009-03-02 15:19:51.587 2009-03-02 15:19:51.587 59392 1 9502745 8913230 2009-03-03 08:13:45.377 2009-03-03 08:13:45.377 293888 1 5407177 5374441 9502734 9371824 2009-02-23 10:09:14.157 2009-02-23 10:11:59.273 6889 1 9502735 9371824 2009-02-23 10:51:41.707 2009-02-23 10:51:41.707 56948 1 9502736 9371824 2009-02-23 10:53:09.610 2009-02-23 10:53:09.610 53627 1 9502737 1179693 2009-02-23 13:02:03.950 2009-02-23 13:02:03.950 770664 1 9502731 1179693 2009-02-23 09:03:15.613 2014-09-04 16:25:02.647 2472 10 8913265 8946018 51 2007-06-08 14:58:06.293 2009-02-05 10:44:13.677 1179693 1933325 1 1179693 2007-10-03 12:33:15.017 2007-10-03 12:33:15.017 9502733 9371824 2009-02-23 10:09:14.150 2009-02-23 10:12:17.057 28515 1 9502732 9371824 2009-02-23 10:09:14.087 2009-02-23 10:12:39.793 26279 1 8913261 8946014 4 2009-02-04 09:06:26.687 2009-02-04 09:13:10.523 8913216 1933326 1 1179926 2007-10-03 12:33:15.020 2007-10-03 12:33:15.020 10355428 10388159 67 2009-03-05 15:41:25.080 2009-07-16 13:22:30.480 9372068 8913270 8946023 1 2009-02-04 13:47:04.580 2009-02-04 13:47:04.580 8913230 8913267 8946020 5 2009-02-04 09:06:26.687 2009-02-05 16:32:29.703 8913216 11796648 11829407 6 2009-03-12 15:26:12.667 2010-04-05 11:25:05.093 9372235 10355371 10388102 65 2009-03-05 15:41:25.080 2009-07-13 16:36:09.507 9372068 11796644 11829403 4 2009-03-12 15:26:12.667 2009-05-27 10:43:31.087 9372235 11796646 11829405 5 2009-03-12 15:26:12.667 2010-04-05 11:21:49.710 9372235 9502770 8913230 2009-03-04 12:28:00.283 2009-03-04 12:28:00.283 412999 1 9502771 8913230 2009-03-04 12:48:49.777 2009-03-04 12:48:49.777 3179593 1 3014678 1 1179693 2008-01-03 16:23:15.073 2008-01-03 16:23:15.073 9798444 9831195 43 2009-03-05 15:41:25.080 2009-05-11 16:47:31.487 9372068 10355402 10388133 66 2009-03-05 15:41:25.080 2009-07-13 16:36:57.613 9372068 9798449 9831200 44 2009-03-05 15:41:25.080 2009-05-12 16:47:10.993 9372068 33652787 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 8913221 8945975 3 2009-02-04 09:06:26.687 2009-02-04 09:12:49.657 8913216 33652786 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 33652785 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 8913219 8945973 1 2009-02-04 09:06:26.687 2009-02-04 09:06:26.687 8913216 33652784 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 8913220 8945974 2 2009-02-04 09:06:26.687 2009-02-04 09:12:03.920 8913216 33652783 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 33652782 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 16777273 16810041 27 2009-02-04 09:06:26.687 2011-04-15 08:11:19.253 8913216 8913223 8945976 47 2007-06-08 14:58:06.293 2009-02-04 08:26:42.090 1179693 33652781 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 33652780 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 16777271 16810039 26 2009-02-04 09:06:26.687 2011-03-23 15:56:07.097 8913216 5374727 5407458 35 2007-06-08 14:58:06.293 2008-07-08 10:01:07.893 1179693 33652779 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 8913213 8945969 1 2009-02-04 08:24:54.600 2009-02-04 08:24:54.600 8913211 33652778 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 33652777 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 8913212 8945968 46 2007-06-08 14:58:06.293 2009-02-04 08:23:09.583 1179693 33652776 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 33652775 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 33652774 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 31621186 9372076 2019-02-25 07:29:03.437 2019-02-25 07:29:03.437 5374725 5407456 34 2007-06-08 14:58:06.293 2008-06-30 15:50:20.727 1179693 33652773 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 33652772 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 33652770 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 33652771 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 10355240 10387973 60 2009-03-05 15:41:25.080 2009-06-26 14:35:21.217 9372068 33652768 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 33652769 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 33587202 33619970 6 2009-02-23 10:11:31.067 2009-02-23 10:55:52.167 9371824 9372408 9405140 82 2007-06-08 14:58:06.293 2009-03-16 08:28:06.167 1179693 33652766 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 8913209 8945965 45 2007-06-08 14:58:06.293 2009-02-04 08:11:07.097 1179693 33652767 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 9372410 9405142 83 2007-06-08 14:58:06.293 2009-03-31 08:22:48.550 1179693 33652764 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 8913207 8945963 44 2007-06-08 14:58:06.293 2009-02-03 14:02:05.593 1179693 33652765 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 33652762 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 33652763 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 33652760 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 16777275 16810043 28 2009-02-04 09:06:26.687 2011-04-15 08:14:45.023 8913216 33652761 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 33652758 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 9502922 8913162 2009-02-02 16:30:52.167 2009-02-04 23:36:14.667 8575 4 9076758 33587204 33619972 116 2007-06-08 14:58:06.293 2020-04-22 17:07:23.480 1179693 33652759 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 33652756 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 33652757 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 33652753 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 33652752 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 33652755 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 33652754 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 33652749 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 33652748 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 33652751 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 10355187 10387922 59 2009-03-05 15:41:25.080 2009-06-24 14:58:45.647 9372068 33652750 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 33652745 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 33652744 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 33652747 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 33652746 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 33652741 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 33652740 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 9798533 9831279 3 2009-03-05 15:48:36.407 2009-03-19 15:10:52.697 9372076 33652743 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 33652742 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 33652737 9371824 2022-02-17 08:39:10.617 2022-02-17 08:39:10.617 33652738 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 33652739 1179693 2022-02-17 08:40:36.530 2022-02-17 08:40:36.530 21365738 21365241 21398451 21434745 2 2013-08-05 16:19:34.827 2013-08-05 16:26:02.927 21365740 21397973 21365241 8913241 8945994 50 2007-06-08 14:58:06.293 2009-02-04 23:38:46.303 1179693 8913232 8945985 48 2007-06-08 14:58:06.293 2009-02-04 09:32:57.450 1179693 8913234 8945987 2 2009-02-02 16:30:29.693 2009-02-02 16:30:52.170 8913162 8913233 8945986 49 2007-06-08 14:58:06.293 2009-02-04 23:06:34.227 1179693 10355284 10388016 85 2007-06-08 14:58:06.293 2009-07-08 11:20:43.557 1179693 8913159 8945917 40 2007-06-08 14:58:06.293 2008-07-15 17:23:29.683 1179693 10355285 10388017 86 2007-06-08 14:58:06.293 2009-07-08 11:29:41.157 1179693 8913160 8945918 41 2007-06-08 14:58:06.293 2009-02-02 16:27:35.913 1179693 11763726 1179693 2010-03-15 16:10:25.790 2010-03-18 13:59:41.507 329192 7 11403360 10355289 10388021 87 2007-06-08 14:58:06.293 2009-07-08 11:33:37.823 1179693 10355279 10388011 62 2009-03-05 15:41:25.080 2009-07-01 16:49:17.157 9372068 10355281 10388013 84 2007-06-08 14:58:06.293 2009-03-31 08:24:14.450 1179693 9798609 9831353 19 2009-02-04 13:47:04.580 2009-04-28 07:48:50.270 8913230 8945919 8913162 10355291 10388023 88 2007-06-08 14:58:06.293 2009-07-08 11:35:04.753 1179693 29687877 29720644 37 2009-02-04 09:06:26.687 2016-12-20 07:28:56.480 8913216 29687878 29720645 38 2009-02-04 09:06:26.687 2016-12-20 07:31:00.343 8913216 29687875 29720642 36 2009-02-04 09:06:26.687 2016-11-02 09:18:10.253 8913216 29687879 29720646 39 2009-02-04 09:06:26.687 2016-12-20 07:31:38.180 8913216 8913192 8945949 43 2007-06-08 14:58:06.293 2009-02-03 08:46:41.910 1179693 10355253 10387985 61 2009-03-05 15:41:25.080 2009-06-30 16:50:49.680 9372068 11239968 11272716 3 2009-02-04 08:24:54.600 2009-03-03 08:26:02.333 8913211 17170433 1179693 2011-05-23 10:08:12.900 2011-05-23 15:48:20.730 17408 16 16973885 33652788 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 33652789 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 33652790 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 33652791 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 33652792 1179693 2022-02-17 08:40:36.533 2022-02-17 08:40:36.533 9011208 1441793 1179693 2009-02-05 21:43:36.160 2009-02-05 21:43:36.160 8913163 8945920 1 2009-02-02 16:30:29.693 2009-02-02 16:30:29.693 8913162 9011207 9142275 1179693 2009-02-04 07:57:57.457 2009-02-04 07:57:57.457 8913165 8945922 42 2007-06-08 14:58:06.293 2009-02-02 16:29:09.040 1179693 33292290 9372235 2021-06-03 12:47:20.447 2021-06-03 12:47:20.447 33292291 9372235 2021-06-03 12:47:20.447 2021-06-03 12:47:20.447 33292289 9372235 2021-06-03 12:47:20.437 2021-06-03 12:47:20.437 9797829 9830597 34 2009-03-05 15:41:25.080 2009-04-28 09:18:10.363 9372068 9797830 9830598 35 2009-03-05 15:41:25.080 2009-04-28 14:43:57.487 9372068 22511642 22544410 17 2009-03-12 15:26:12.667 2013-08-12 16:16:53.137 9372235 9797824 9830592 33 2009-03-05 15:41:25.080 2009-04-27 16:29:39.070 9372068 9797822 9830590 18 2009-02-04 13:47:04.580 2009-03-10 10:01:13.007 8913230 9404567 9371817 9797817 9830585 32 2009-03-05 15:41:25.080 2009-04-27 15:35:45.730 9372068 1835116 1867880 4 2007-10-03 11:50:19.747 2007-10-03 14:16:47.437 1835085 1835115 1867879 3 2007-10-03 11:50:19.747 2007-10-03 11:55:23.087 1835085 1835118 1867882 6 2007-10-03 11:50:19.747 2007-10-03 15:46:39.763 1835085 1835117 1867881 5 2007-10-03 11:50:19.747 2007-10-03 15:45:25.137 1835085 16416910 16449673 25 2009-02-04 09:06:26.687 2011-03-23 12:34:52.160 8913216 9797808 9830576 31 2009-03-05 15:41:25.080 2009-04-23 17:03:26.713 9372068 10355011 10387764 53 2009-03-05 15:41:25.080 2009-06-09 15:21:24.887 9372068 33292292 9372235 2021-06-03 12:47:20.447 2021-06-03 12:47:20.447 9404574 9371824 16416904 16449667 4 2009-02-04 08:24:54.600 2010-02-18 11:50:59.260 8913211 16416906 16449669 24 2009-02-04 09:06:26.687 2009-12-22 16:04:37.777 8913216 14516242 14549009 25 2009-02-04 13:47:04.580 2009-11-11 10:35:22.060 8913230 9371691 9404455 9 2009-02-04 09:06:26.687 2009-02-18 16:55:48.313 8913216 9798698 9831437 3 2009-03-12 15:26:12.667 2009-03-12 15:38:00.023 9372235 14516247 14549014 27 2009-02-04 13:47:04.580 2010-08-24 16:30:30.513 8913230 14516243 14549010 26 2009-02-04 13:47:04.580 2010-08-24 16:30:05.187 8913230 14516245 14549012 5 2009-03-05 15:48:36.407 2009-06-22 13:49:44.270 9372076 9797765 9830534 29 2009-03-05 15:41:25.080 2009-04-20 16:40:29.340 9372068 9797764 9830533 28 2009-03-05 15:41:25.080 2009-04-20 15:10:48.560 9372068 21433176 21365241 2013-06-27 08:40:33.950 2013-06-27 08:40:33.950 9371687 9404451 6 2009-02-04 09:06:26.687 2009-02-05 16:44:23.897 8913216 9797758 9830527 26 2009-03-05 15:41:25.080 2009-04-09 15:01:49.360 9372068 21433177 21365241 2013-06-27 08:40:33.950 2013-06-27 08:40:33.950 9371689 9404453 7 2009-02-04 09:06:26.687 2009-02-18 16:52:44.470 8913216 9371690 9404454 8 2009-02-04 09:06:26.687 2009-02-18 16:55:32.603 8913216 9371683 9404447 55 2007-06-08 14:58:06.293 2009-02-06 18:05:50.637 1179693 9797762 9830531 27 2009-03-05 15:41:25.080 2009-04-20 13:12:22.010 9372068 9371684 9404448 56 2007-06-08 14:58:06.293 2009-02-18 15:36:43.120 1179693 9371685 9404449 1 2009-02-05 16:40:27.063 2009-02-05 16:40:27.063 8913263 21433175 21365241 2013-06-27 08:40:33.950 2013-06-27 08:40:33.950 32407555 32440323 115 2007-06-08 14:58:06.293 2019-02-25 07:28:11.003 1179693 29196540 29229308 35 2009-02-04 09:06:26.687 2016-11-02 09:16:20.400 8913216 31522820 1 9372235 2019-04-05 09:52:15.053 2019-04-05 09:52:15.053 3276915 1 1179693 2008-02-05 10:49:15.063 2008-02-05 10:49:15.063 9797794 9830563 30 2009-03-05 15:41:25.080 2009-04-20 16:41:05.710 9372068 1179694 1212462 1 2007-06-08 14:58:06.293 2007-06-08 14:58:06.293 1179693 1179698 1212466 5 2007-06-08 14:58:06.293 2007-06-08 16:36:40.160 1179693 1179697 1212465 4 2007-06-08 14:58:06.293 2007-06-08 16:34:12.107 1179693 1179696 1212464 3 2007-06-08 14:58:06.293 2007-06-08 15:00:04.157 1179693 3113130 3178651 30 2007-06-08 14:58:06.293 2008-01-10 14:22:33.630 1179693 1179695 1212463 2 2007-06-08 14:58:06.293 2007-06-08 14:59:41.433 1179693 1179702 1212470 9 2007-06-08 14:58:06.293 2007-06-08 17:00:10.757 1179693 29196532 29229300 30 2009-02-04 09:06:26.687 2012-12-17 13:57:00.337 8913216 1179701 1212469 8 2007-06-08 14:58:06.293 2007-06-08 16:58:25.343 1179693 1179700 1212468 7 2007-06-08 14:58:06.293 2007-06-08 16:39:02.220 1179693 29196534 29229302 31 2009-02-04 09:06:26.687 2016-11-02 08:35:43.067 8913216 1179699 1212467 6 2007-06-08 14:58:06.293 2007-06-08 16:37:32.227 1179693 29196536 29229304 32 2009-02-04 09:06:26.687 2016-11-02 08:49:43.893 8913216 29196537 29229305 33 2009-02-04 09:06:26.687 2016-11-02 09:14:38.847 8913216 1179704 1212472 11 2007-06-08 14:58:06.293 2007-06-11 09:31:48.820 1179693 29196538 29229306 34 2009-02-04 09:06:26.687 2016-11-02 09:15:23.127 8913216 1179703 1212471 10 2007-06-08 14:58:06.293 2007-06-10 20:21:11.117 1179693 16416798 16449565 28 2009-02-04 13:47:04.580 2010-08-24 16:35:24.097 8913230 21365293 21398020 35 2013-05-06 15:48:48.900 2013-05-20 10:48:25.413 20971983 10355129 10387866 58 2009-03-05 15:41:25.080 2009-06-23 15:48:37.593 9372068 21365297 21398024 37 2013-05-06 15:48:48.900 2013-07-02 13:49:16.360 20971983 16416800 16449567 29 2009-02-04 13:47:04.580 2011-03-15 08:13:38.857 8913230 21365295 21398022 36 2013-05-06 15:48:48.900 2013-07-02 12:53:17.443 20971983 21365301 21398028 39 2013-05-06 15:48:48.900 2013-07-02 13:56:08.280 20971983 21365299 21398026 38 2013-05-06 15:48:48.900 2013-07-02 13:54:24.467 20971983 21365305 21398032 41 2013-05-06 15:48:48.900 2013-07-02 14:28:50.977 20971983 21365303 21398030 40 2013-05-06 15:48:48.900 2013-07-02 14:23:01.810 20971983 21365309 21398036 43 2013-05-06 15:48:48.900 2013-07-02 15:04:03.960 20971983 21365307 21398034 42 2013-05-06 15:48:48.900 2013-07-02 14:35:26.147 20971983 21365311 21398038 44 2013-05-06 15:48:48.900 2013-07-02 15:28:10.487 20971983 9371851 9404601 16 2009-02-04 09:06:26.687 2009-02-24 09:07:48.787 8913216 9371852 9404602 17 2009-02-04 09:06:26.687 2009-02-24 09:12:09.097 8913216 9371854 9404604 18 2009-02-04 09:06:26.687 2009-02-24 09:14:31.333 8913216 30703906 30736642 42 2009-02-04 09:06:26.687 2018-11-30 11:46:07.240 8913216 9371856 9404606 19 2009-02-04 09:06:26.687 2009-02-24 09:23:40.210 8913216 30703904 30736640 41 2009-02-04 09:06:26.687 2017-05-25 07:34:30.107 8913216 9371846 9404596 13 2009-02-04 09:06:26.687 2009-02-24 09:02:25.690 8913216 9371845 9404595 12 2009-02-04 09:06:26.687 2009-02-24 08:57:28.340 8913216 9371843 9404593 11 2009-02-04 09:06:26.687 2009-02-24 08:56:57.230 8913216 9371849 9404599 15 2009-02-04 09:06:26.687 2009-02-24 09:05:17.270 8913216 9371847 9404597 14 2009-02-04 09:06:26.687 2009-02-24 09:03:54.317 8913216 9371837 9404587 64 2007-06-08 14:58:06.293 2009-02-23 10:14:34.637 1179693 9371835 9404585 5 2009-02-23 10:11:31.067 2009-02-23 10:53:32.013 9371824 21365333 21398059 45 2013-05-06 15:48:48.900 2013-07-02 15:38:13.333 20971983 9371842 9404592 10 2009-02-04 09:06:26.687 2009-02-18 16:56:27.273 8913216 3113043 3178575 29 2007-06-08 14:58:06.293 2008-01-03 14:45:07.677 1179693 21365335 21398061 46 2013-05-06 15:48:48.900 2013-07-03 13:37:07.693 20971983 9371830 9404580 2 2009-02-23 10:11:31.067 2009-02-23 10:20:07.593 9371824 9371828 9404578 1 2009-02-23 10:11:31.067 2009-02-23 10:11:31.067 9371824 9371834 9404584 4 2009-02-23 10:11:31.067 2009-02-23 10:24:32.287 9371824 9371832 9404582 3 2009-02-23 10:11:31.067 2009-02-23 10:21:57.603 9371824 9371822 9404572 62 2007-06-08 14:58:06.293 2009-02-23 09:00:22.447 1179693 9371819 9404569 60 2007-06-08 14:58:06.293 2009-02-23 08:56:39.657 1179693 9371820 9404570 61 2007-06-08 14:58:06.293 2009-02-23 08:59:12.303 1179693 9371826 9404576 63 2007-06-08 14:58:06.293 2009-02-23 09:03:45.797 1179693 1835078 1867842 23 2007-06-08 14:58:06.293 2007-08-02 14:19:35.253 1179693 9371818 9404568 59 2007-06-08 14:58:06.293 2009-02-20 16:55:32.757 1179693 10355062 10387802 54 2009-03-05 15:41:25.080 2009-06-17 14:08:43.027 9372068 10356042 10388768 89 2007-06-08 14:58:06.293 2009-07-08 11:37:53.207 1179693 21365381 21365241 21398107 1 2013-07-10 16:55:16.807 2013-07-10 16:55:16.807 26640696 26673453 111 2007-06-08 14:58:06.293 2013-05-03 21:12:06.270 1179693 1933366 1 1835085 2007-10-03 16:44:15.030 2007-10-03 16:44:15.030 16416869 16449632 30 2009-02-04 13:47:04.580 2011-03-15 08:18:39.300 8913230 1933367 1 1179693 2007-10-04 11:07:15.023 2007-10-04 11:07:15.023 27557907 27590675 35 2009-02-04 13:47:04.580 2013-09-10 08:54:21.093 8913230 1179873 1212638 12 2007-06-08 14:58:06.293 2007-06-11 09:36:09.097 1179693 21365250 21397981 3 2013-06-27 08:00:20.133 2013-06-27 08:24:14.550 21365241 10355077 10387815 55 2009-03-05 15:41:25.080 2009-06-18 15:35:13.747 9372068 21365248 21397979 2 2013-06-27 08:00:20.133 2013-06-27 08:21:33.463 21365241 21365245 21397976 1 2013-06-27 08:00:20.133 2013-06-27 08:00:20.133 21365241 9371795 9404550 58 2007-06-08 14:58:06.293 2009-02-20 16:53:32.660 1179693 10355079 10387817 4 2009-03-05 15:48:36.407 2009-05-14 13:16:39.660 9372076 10355081 10387819 20 2009-02-04 13:47:04.580 2009-05-20 08:59:39.980 8913230 26640699 26673456 112 2007-06-08 14:58:06.293 2014-09-04 16:25:40.767 1179693 9371794 9404549 57 2007-06-08 14:58:06.293 2009-02-18 16:48:29.240 1179693 21365253 21397984 5 2013-06-27 08:00:20.133 2013-06-27 08:40:04.840 21365241 21365252 21397983 4 2013-06-27 08:00:20.133 2013-06-27 08:31:11.237 21365241 9797662 9830430 25 2009-03-05 15:41:25.080 2009-03-17 14:51:19.407 9372068 10355097 10387834 57 2009-03-05 15:41:25.080 2009-06-22 15:59:24.640 9372068 10355084 10387822 56 2009-03-05 15:41:25.080 2009-06-22 13:29:01.803 9372068 1343507 1 1179693 2007-06-08 16:53:15.017 2007-06-08 16:53:15.017 1343508 5 1179693 2007-06-08 17:01:15.033 2008-01-03 16:23:15.070 1343509 1 1179693 2007-06-12 16:10:15.067 2007-06-12 16:10:15.067 1835086 1867850 1 2007-10-03 11:50:19.747 2007-10-03 11:50:19.747 1835085 10356047 10388773 92 2007-06-08 14:58:06.293 2009-08-19 16:09:47.087 1179693 10356046 10388772 91 2007-06-08 14:58:06.293 2009-08-19 16:08:40.170 1179693 1343503 1 1179693 2007-06-08 14:58:15.023 2007-06-08 14:58:15.023 1343504 8 1179693 2007-06-08 15:00:15.027 2007-06-08 17:00:15.040 10356044 10388770 90 2007-06-08 14:58:06.293 2009-08-19 16:05:39.640 1179693 1835087 1867851 2 2007-10-03 11:50:19.747 2007-10-03 11:50:56.617 1835085 1343505 8 1179693 2007-06-08 16:34:15.043 2008-01-03 16:22:15.123 1835088 1867852 24 2007-06-08 14:58:06.293 2007-10-03 10:39:52.240 1179693 21365339 20971983 21398065 1 2013-07-03 16:33:58.463 2013-07-03 16:33:58.463 9371986 9404735 5 2009-02-04 13:47:04.580 2009-03-03 08:19:19.323 8913230 19169597 1179693 2012-12-17 13:33:48.213 2012-12-17 13:33:48.213 133 1 9404822 9372076 9371984 9404733 4 2009-02-04 13:47:04.580 2009-03-02 15:34:56.137 8913230 1179930 1212695 16 2007-06-08 14:58:06.293 2007-08-01 12:14:30.947 1179693 1179929 1212694 15 2007-06-08 14:58:06.293 2007-08-01 11:58:11.123 1179693 9371989 9404738 1 2009-02-03 09:34:08.390 2009-02-03 09:34:08.390 8913167 9371988 9404737 2 2009-02-04 08:24:54.600 2009-02-04 08:27:03.700 8913211 1179928 1212693 2 2007-08-01 12:00:41.590 2007-08-01 12:10:10.737 1179926 1179927 1212692 1 2007-08-01 12:00:41.590 2007-08-01 12:00:41.590 1179926 1179925 1212690 14 2007-06-08 14:58:06.293 2007-07-30 14:24:39.307 1179693 9371997 9404746 6 2009-02-04 13:47:04.580 2009-03-03 08:23:50.747 8913230 1179937 1212702 19 2007-06-08 14:58:06.293 2007-08-01 12:52:04.570 1179693 1179935 1212700 17 2007-06-08 14:58:06.293 2007-08-01 12:16:19.457 1179693 32178179 32210946 46 2009-02-04 09:06:26.687 2019-02-25 07:51:39.853 8913216 1179936 1212701 18 2007-06-08 14:58:06.293 2007-08-01 12:41:16.100 1179693 9372001 9404750 1 2009-03-03 08:49:09.297 2009-03-03 08:49:09.297 9371999 1179933 1212698 5 2007-08-01 12:00:41.590 2007-08-01 12:21:44.620 1179926 1179934 1212699 6 2007-08-01 12:00:41.590 2007-08-01 12:28:26.377 1179926 1179931 1212696 3 2007-08-01 12:00:41.590 2007-08-01 12:10:56.290 1179926 1179932 1212697 4 2007-08-01 12:00:41.590 2007-08-01 12:19:35.920 1179926 9372005 9404753 67 2007-06-08 14:58:06.293 2009-03-03 08:53:14.477 1179693 9372003 9404751 66 2007-06-08 14:58:06.293 2009-03-02 16:04:55.237 1179693 9404814 9372068 1179941 1212706 21 2007-06-08 14:58:06.293 2007-08-02 12:23:32.187 1179693 1179942 1212707 22 2007-06-08 14:58:06.293 2007-08-02 14:18:43.657 1179693 1179939 1212704 20 2007-06-08 14:58:06.293 2007-08-01 15:59:39.330 1179693 20087083 20119844 33 2009-02-04 13:47:04.580 2013-02-28 16:51:15.917 8913230 10485976 1179693 2009-09-28 16:59:05.657 2009-09-28 16:59:05.657 119808 1 10485977 1179693 2009-08-19 16:02:00.987 2009-08-19 16:02:00.987 328362 1 10485952 9371961 9404710 2 2009-02-04 13:47:04.580 2009-02-05 21:09:40.227 8913230 1179893 1212658 13 2007-06-08 14:58:06.293 2007-07-19 15:41:44.737 1179693 10485953 1179693 2009-02-06 18:02:08.920 2009-02-06 18:04:46.713 2532 2 9076787 10485952 1179693 2009-08-19 16:02:00.987 2009-09-28 17:26:29.247 328698 2 9371965 9404714 3 2009-02-04 13:47:04.580 2009-03-02 15:11:30.343 8913230 9371968 9404717 20 2009-02-04 09:06:26.687 2009-02-24 09:47:44.497 8913216 10485955 1179693 2009-02-23 09:03:15.613 2009-07-08 11:28:35.290 2394 3 9502731 9371970 9404719 2 2009-02-05 16:40:27.063 2009-02-18 16:48:29.337 8913263 10485954 1179693 2009-02-06 18:02:08.907 2009-02-06 18:02:08.907 1707 1 9076786 9371971 9404720 65 2007-06-08 14:58:06.293 2009-02-23 13:03:17.627 1179693 9371974 9404723 1 2009-03-02 16:47:50.330 2009-03-02 16:47:50.330 9371973 9371976 9404725 21 2009-02-04 09:06:26.687 2009-03-02 15:58:31.893 8913216 9371977 9404726 22 2009-02-04 09:06:26.687 2009-03-02 16:53:03.770 8913216 22282301 22315069 34 2009-02-04 13:47:04.580 2013-02-28 16:51:55.070 8913230 21037073 1179693 2013-05-03 21:08:08.787 2013-05-03 21:08:08.787 338795 1 20972074 21004837 26 2013-05-06 15:48:48.900 2013-05-14 11:34:18.013 20971983 21037074 1179693 2009-02-06 18:02:08.907 2013-01-15 16:39:17.093 2380 8 9076786 21037075 1179693 2009-02-23 09:03:15.613 2013-01-15 16:23:38.490 2471 8 9502731 21037076 1179693 2009-02-06 18:02:08.920 2013-01-10 16:15:42.193 4009 8 9076787 20972059 21004822 23 2013-05-06 15:48:48.900 2013-05-13 09:31:19.767 20971983 31424514 4 1179693 2019-01-29 09:07:15.357 2019-01-29 09:08:15.020 20972060 21004823 24 2013-05-06 15:48:48.900 2013-05-14 08:58:11.193 20971983 20972062 21004825 25 2013-05-06 15:48:48.900 2013-05-14 08:59:00.640 20971983 9798021 9830777 40 2009-03-05 15:41:25.080 2009-04-30 09:21:59.417 9372068 20087082 20119843 32 2009-02-04 13:47:04.580 2012-12-17 13:53:21.797 8913230 11403380 1179693 2010-03-15 16:10:25.790 2010-03-18 13:39:52.467 329212 6 11403360 1179692 1409026 1212460 2 2007-06-08 14:58:06.287 2009-02-05 23:30:16.317 8913275 12025859 29523970 31457281 20972042 21004805 22 2013-05-06 15:48:48.900 2013-05-10 15:40:09.110 20971983 11403379 1179693 2010-03-15 16:10:25.790 2010-03-16 15:54:01.663 329215 5 11403360 20972035 21004798 21 2013-05-06 15:48:48.900 2013-05-10 15:33:54.110 20971983 20972033 21004796 20 2013-05-06 15:48:48.900 2013-05-08 17:58:32.377 20971983 16547846 8913230 2011-03-15 08:14:15.073 2011-03-15 08:14:15.073 447279 1 19169615 1179693 2009-02-23 09:03:15.613 2013-01-10 16:15:42.187 1920 7 9502731 19169614 1179693 2009-02-06 18:02:08.907 2013-01-10 16:37:18.843 2050 6 9076786 19169617 1179693 2009-02-06 18:02:08.907 2013-01-15 16:23:38.450 2380 7 9076786 20972027 21004790 19 2013-05-06 15:48:48.900 2013-05-08 17:58:05.537 20971983 19169616 1179693 2013-01-15 16:38:37.517 2013-01-15 16:38:37.517 338809 1 19169610 1179693 2009-02-06 18:02:08.920 2012-01-12 14:00:22.160 3873 7 9076787 20972026 21004789 18 2013-05-06 15:48:48.900 2013-05-08 17:54:13.780 20971983 11403364 1179693 2009-02-06 18:02:08.920 2010-03-15 16:12:25.010 3080 4 9076787 19169612 1179693 2009-02-06 18:02:08.907 2013-01-10 16:15:42.157 2050 5 9076786 11403363 1179693 2009-02-06 18:02:08.920 2009-08-19 16:06:02.743 2756 3 9076787 18841601 1179693 2011-07-22 12:52:07.867 2011-07-22 12:52:07.867 329422 1 18186263 20972024 21004787 17 2013-05-06 15:48:48.900 2013-05-08 17:52:01.117 20971983 11403362 1179693 2009-02-23 09:03:15.613 2009-08-19 16:06:36.057 2578 4 9502731 10485890 1179693 2009-02-23 09:03:15.613 2009-07-08 11:22:57.827 2389 2 9502731 11403361 1179693 2009-02-06 18:02:08.907 2009-08-19 16:06:36.043 1759 2 9076786 10485889 1179693 2009-02-23 09:03:15.613 2009-02-23 09:03:46.007 2384 1 9502731 20972022 21004785 16 2013-05-06 15:48:48.900 2013-05-08 17:23:08.820 20971983 18841602 1179693 2009-02-06 18:02:08.920 2011-07-21 10:51:40.597 3755 6 9076787 11403360 1179693 2010-03-15 16:10:25.790 2010-04-05 17:30:27.307 329259 8 19169608 1179693 2009-02-06 18:02:08.907 2011-07-21 10:51:40.593 1771 4 9076786 20972020 21004783 15 2013-05-06 15:48:48.900 2013-05-08 17:17:56.557 20971983 19169609 1179693 2009-02-23 09:03:15.613 2011-07-21 10:51:40.567 2151 6 9502731 20972018 21004781 14 2013-05-06 15:48:48.900 2013-05-07 16:21:14.510 20971983 11403369 1179693 2010-03-15 16:10:25.790 2010-03-16 15:40:03.333 329238 4 11403360 19169598 1179693 2011-05-23 10:08:12.900 2011-05-25 07:56:10.213 18301 17 16973885 11403368 1179693 2010-03-15 16:10:25.790 2010-03-16 15:00:56.607 329241 3 11403360 19169599 1179693 2011-05-23 10:08:12.900 2012-12-17 13:36:54.220 18008 18 16973885 11403367 1179693 2010-03-15 16:10:25.790 2010-03-16 13:43:50.990 329164 2 11403360 11403366 1179693 2010-03-15 16:10:25.790 2010-03-15 16:10:25.790 329050 1 11403360 9372117 9404862 5 2009-03-02 16:47:50.330 2009-03-04 12:56:55.923 9371973 9797936 9830703 37 2009-03-05 15:41:25.080 2009-04-29 14:20:19.297 9372068 9372115 9404860 13 2009-02-04 13:47:04.580 2009-03-09 10:30:42.677 8913230 5767179 1179693 2008-07-15 17:08:25.073 2008-07-15 17:08:25.073 23040 1 9372110 9404855 8 2009-03-05 15:41:25.080 2009-03-09 09:49:31.460 9372068 10354879 10387643 47 2009-03-05 15:41:25.080 2009-06-03 14:29:48.800 9372068 9372108 9404853 7 2009-03-05 15:41:25.080 2009-03-09 08:36:29.973 9372068 9372113 9404858 12 2009-02-04 13:47:04.580 2009-03-05 15:44:29.297 8913230 10354877 10387641 46 2009-03-05 15:41:25.080 2009-06-01 17:20:57.330 9372068 5767170 1179693 2008-07-14 12:09:51.397 2008-07-14 12:09:51.397 15881 1 9372111 9404856 77 2007-06-08 14:58:06.293 2009-03-05 16:07:03.547 1179693 9372134 9404877 9 2009-03-05 15:41:25.080 2009-03-09 09:50:46.390 9372068 9372132 9404875 80 2007-06-08 14:58:06.293 2009-03-09 12:05:42.360 1179693 9372138 9404881 10 2009-03-05 15:41:25.080 2009-03-09 12:10:15.957 9372068 9797949 9830716 38 2009-03-05 15:41:25.080 2009-04-29 14:37:45.307 9372068 5767182 1179693 2008-07-16 13:21:34.220 2008-07-16 13:21:34.220 310272 1 26771530 2 1179693 2014-09-25 10:46:15.030 2014-11-10 12:10:15.030 26771531 1 1179693 2014-09-25 12:12:15.027 2014-09-25 12:12:15.027 5374439 5407175 31 2007-06-08 14:58:06.293 2008-01-15 11:35:04.900 1179693 16973885 1179693 2011-05-23 10:08:12.900 2012-12-17 13:37:25.430 18009 19 9372129 9404873 79 2007-06-08 14:58:06.293 2009-03-09 12:04:46.943 1179693 10354859 10387623 45 2009-03-05 15:41:25.080 2009-05-12 17:12:53.753 9372068 9372128 9404872 78 2007-06-08 14:58:06.293 2009-03-09 10:16:45.567 1179693 9797969 9830735 39 2009-03-05 15:41:25.080 2009-04-29 15:38:26.347 9372068 10354919 10387684 52 2009-03-05 15:41:25.080 2009-06-08 15:15:09.653 9372068 32243713 7 1179693 2019-11-01 16:44:15.120 2020-09-04 14:14:15.050 9372087 9404833 7 2009-03-03 08:49:09.297 2009-03-05 15:46:57.647 9371999 26837031 1179693 2014-09-04 16:20:37.673 2014-09-04 16:20:37.673 338853 1 26837027 10354915 10387680 50 2009-03-05 15:41:25.080 2009-06-08 13:35:49.510 9372068 26837030 1179693 2009-02-06 18:02:08.920 2013-05-03 21:10:39.327 4105 9 9076787 26837033 1179693 2009-02-06 18:02:08.907 2014-09-04 16:25:02.607 2381 10 9076786 10354917 10387682 51 2009-03-05 15:41:25.080 2009-06-08 13:41:57.250 9372068 26837032 1179693 2014-09-04 16:20:37.673 2014-09-05 10:36:36.487 338813 2 26837027 26837027 1179693 2014-09-04 16:20:37.673 2014-09-05 11:09:16.573 338827 3 10354914 10387679 49 2009-03-05 15:41:25.080 2009-06-04 15:49:50.357 9372068 26837029 1179693 2009-02-23 09:03:15.613 2013-05-03 21:10:39.310 2472 9 9502731 9372078 9404824 75 2007-06-08 14:58:06.293 2009-03-05 15:46:57.630 1179693 26837028 1179693 2009-02-06 18:02:08.907 2013-05-03 21:10:39.260 2381 9 9076786 9372081 9404827 2 2009-03-05 15:41:25.080 2009-03-05 15:42:04.857 9372068 9372082 9404828 76 2007-06-08 14:58:06.293 2009-03-05 15:50:09.990 1179693 9372100 9404846 5 2009-03-05 15:41:25.080 2009-03-06 16:17:50.687 9372068 10354904 10387668 48 2009-03-05 15:41:25.080 2009-06-03 17:08:35.080 9372068 9372102 9404848 6 2009-03-05 15:41:25.080 2009-03-09 08:32:19.697 9372068 9404722 9371973 9372092 9404838 3 2009-03-05 15:41:25.080 2009-03-05 16:02:23.043 9372068 9076800 1179693 2009-02-06 17:58:27.180 2009-02-09 09:01:56.570 325659 2 9076785 9372096 9404842 4 2009-03-05 15:41:25.080 2009-03-06 14:09:48.467 9372068 9372098 9404844 8 2009-03-03 08:49:09.297 2009-03-06 09:28:40.763 9371999 18186263 1179693 2011-07-22 12:52:07.867 2012-01-12 13:59:25.677 329496 2 18186262 1179693 2009-02-06 18:02:08.920 2010-03-15 16:14:00.817 3080 5 9076787 9372053 9404799 3 2009-03-02 16:47:50.330 2009-03-04 08:09:46.077 9371973 9372054 9404800 4 2009-03-02 16:47:50.330 2009-03-04 12:56:31.737 9371973 9372051 9404797 10 2009-02-04 13:47:04.580 2009-03-04 12:43:54.137 8913230 4849687 1 1179693 2008-06-03 13:16:15.047 2008-06-03 13:16:15.047 9076791 1179693 2009-02-06 17:58:27.180 2009-02-06 17:58:27.180 325617 1 9076785 4849686 13 1179693 2008-06-03 13:15:15.020 2008-06-05 17:09:15.030 9372049 9404795 9 2009-02-04 13:47:04.580 2009-03-04 12:40:48.593 8913230 9076789 1179693 2009-02-06 18:02:08.920 2009-02-06 18:02:08.920 2532 1 9076787 9372047 9404793 8 2009-02-04 13:47:04.580 2009-03-04 12:35:33.203 8913230 9076787 1179693 2009-02-06 18:02:08.920 2014-09-04 16:25:02.677 4250 10 9076786 1179693 2009-02-06 18:02:08.907 2014-09-05 11:10:09.503 2381 11 9372045 9404791 7 2009-02-04 13:47:04.580 2009-03-03 08:42:26.117 8913230 9076785 1179693 2009-02-06 17:58:27.180 2009-02-09 11:01:22.600 325617 3 18186261 1179693 2009-02-06 18:02:08.907 2010-03-15 16:12:24.953 1758 3 9076786 18186260 1179693 2009-02-23 09:03:15.613 2010-03-15 16:12:25.000 2151 5 9502731 9372074 9404820 6 2009-03-03 08:49:09.297 2009-03-05 13:58:29.043 9371999 9372073 9404819 74 2007-06-08 14:58:06.293 2009-03-05 15:24:10.300 1179693 9404748 9371999 9372071 9404817 11 2009-02-04 13:47:04.580 2009-03-04 12:48:57.457 8913230 9372069 9404815 1 2009-03-05 15:41:25.080 2009-03-05 15:41:25.080 9372068 9372065 9404811 2 2009-03-04 08:19:28.920 2009-03-04 08:53:37.663 9372027 9372064 9404810 73 2007-06-08 14:58:06.293 2009-03-05 15:19:32.753 1179693 9306120 17 1179693 2009-02-20 17:36:15.097 2020-11-25 10:28:15.050 9372062 9404808 72 2007-06-08 14:58:06.293 2009-03-04 08:20:35.043 1179693 9076766 8913162 2009-02-02 16:30:52.167 2009-02-04 10:07:31.550 1353 2 9076758 9372060 9404806 5 2009-03-03 08:49:09.297 2009-03-05 08:09:26.563 9371999 9076767 8913162 2009-02-02 16:30:52.167 2009-02-04 23:18:20.187 3826 3 9076758 9372059 9404805 4 2009-03-03 08:49:09.297 2009-03-04 09:08:04.623 9371999 9306119 1 1179693 2009-02-20 17:05:15.090 2009-02-20 17:05:15.090 9076763 8913162 2009-02-02 16:30:52.167 2009-02-02 16:30:52.167 133 1 9076758 9372023 9404769 2 2009-03-02 16:47:50.330 2009-03-02 16:48:17.127 9371973 4882501 1409026 2008-06-03 12:46:38.077 2008-06-03 12:46:38.077 16973889 1179693 2011-05-23 10:08:12.900 2011-05-23 12:56:01.730 9410 4 16973885 4882502 1409026 2008-06-03 12:46:38.077 2008-06-03 12:46:38.077 9372024 9404770 70 2007-06-08 14:58:06.293 2009-03-04 08:09:02.217 1179693 16973888 1179693 2011-05-23 10:08:12.900 2011-05-23 12:27:32.843 3076 3 16973885 4882503 1409026 2008-06-03 12:46:38.077 2008-06-03 12:46:38.077 16973887 1179693 2011-05-23 10:08:12.900 2011-05-23 10:11:04.437 2764 2 16973885 16973886 1179693 2011-05-23 10:08:12.900 2011-05-23 10:08:12.900 133 1 16973885 16973893 1179693 2011-05-23 10:08:12.900 2011-05-23 15:26:54.480 17214 8 16973885 9076758 8913162 2009-02-02 16:30:52.167 2013-08-12 13:37:05.103 8613 8 16973892 1179693 2011-05-23 10:08:12.900 2011-05-23 15:26:28.520 17193 7 16973885 9372021 9404767 69 2007-06-08 14:58:06.293 2009-03-04 08:06:52.847 1179693 16973891 1179693 2011-05-23 10:08:12.900 2011-05-23 15:22:54.133 17111 6 16973885 16973890 1179693 2011-05-23 10:08:12.900 2011-05-23 15:22:33.000 17124 5 16973885 16973897 1179693 2011-05-23 10:08:12.900 2011-05-23 15:39:54.453 17566 12 16973885 16973896 1179693 2011-05-23 10:08:12.900 2011-05-23 15:38:25.273 17487 11 16973885 16973895 1179693 2011-05-23 10:08:12.900 2011-05-23 15:37:53.897 17715 10 16973885 9372018 9404766 68 2007-06-08 14:58:06.293 2009-03-03 09:04:29.260 1179693 16973894 1179693 2011-05-23 10:08:12.900 2011-05-23 15:37:38.547 17264 9 16973885 16973900 1179693 2011-05-23 10:08:12.900 2011-05-23 15:46:59.027 17378 15 16973885 16973899 1179693 2011-05-23 10:08:12.900 2011-05-23 15:40:59.020 17342 14 16973885 9372014 9404762 2 2009-03-03 08:49:09.297 2009-03-03 08:50:22.467 9371999 16973898 1179693 2011-05-23 10:08:12.900 2011-05-23 15:40:35.563 17380 13 16973885 9372040 9404786 3 2009-03-03 08:49:09.297 2009-03-03 16:21:34.990 9371999 11240432 11273169 93 2007-06-08 14:58:06.293 2009-08-19 16:10:14.123 1179693 9404782 9372036 9797922 9830690 36 2009-03-05 15:41:25.080 2009-04-28 14:48:03.053 9372068 9404773 9372027 9372033 9404779 1 2009-03-04 08:19:28.920 2009-03-04 08:19:28.920 9372027 4882498 1409026 2008-06-03 12:46:38.077 2008-06-03 12:46:38.077 9372028 9404774 71 2007-06-08 14:58:06.293 2009-03-04 08:10:45.437 1179693 4882497 1409026 2008-06-03 12:46:25.630 2008-06-03 12:46:25.630 4882500 1409026 2008-06-03 12:46:38.077 2008-06-03 12:46:38.077 4882499 1409026 2008-06-03 12:46:38.077 2008-06-03 12:46:38.077 8945975 8913221 8945976 8913223 8945973 8913219 8945974 8913220 10388013 10355281 8945985 8913232 8945986 8913233 10388011 10355279 10388017 10355285 10388016 10355284 10388021 10355289 21434745 21365738 2013-08-05 16:26:02.930 2013-08-05 16:26:02.930 10388023 10355291 8945968 8913212 8945969 8913213 8945963 8913207 8945965 8913209 9831353 9798609 10387973 10355240 21434802 21365583 2013-08-07 08:47:07.860 2013-08-07 08:47:07.860 21434803 21365583 2013-08-07 08:47:07.860 2013-08-07 08:47:07.860 8945949 8913192 10387985 10355253 16810041 16777273 16810039 16777271 9405140 9372408 16810043 16777275 9405142 9372410 21365585 21398305 1 2013-07-23 15:22:05.897 2013-07-23 15:22:05.897 21365582 21365584 21398304 1 2013-07-23 15:23:20.427 2013-07-23 15:23:20.427 21365583 32210946 32178179 21365573 21398293 4 2013-07-23 09:42:27.857 2013-07-23 11:29:36.407 21365564 21365572 21398292 3 2013-07-23 09:42:27.857 2013-07-23 11:29:00.190 21365564 21365571 21398291 2 2013-07-23 09:42:27.857 2013-07-23 11:27:56.487 21365564 8946020 8913267 21365570 21398290 1 2013-07-23 09:42:27.857 2013-07-23 09:42:27.857 21365564 21365565 21398285 1 2013-07-17 17:16:11.377 2013-07-17 17:16:11.377 21365524 21365566 21398286 2 2013-07-17 17:16:11.377 2013-07-23 10:29:46.893 21365524 8946023 8913270 27590675 27557907 9405070 9372337 21365548 21398268 1 2013-07-19 14:45:12.383 2013-07-19 14:45:12.383 21365547 1212462 1179694 8946014 8913261 1212460 1179692 1212466 1179698 8946018 8913265 1212465 1179697 1212464 1179696 1212463 1179695 1212469 1179701 21397518 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 h4. 16 May 2013 Another post to the pgsql-perform mail list asking for opinions on a more conventional server configuration: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 3:36:03 PM PDT To: pgsql-performance@postgresql.org Cc: Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On May 7, 2013, at 4:21 PM, Jeff Janes wrote: On Thu, May 2, 2013 at 6:35 PM, Scott Marlowe wrote: On Thu, May 2, 2013 at 5:11 PM, Mike McCann wrote: > Hello, > > We are in the fortunate situation of having more money than time to help > solve our PostgreSQL 9.1 performance problem. > > Our server hosts databases that are about 1 GB in size with the largest > tables having order 10 million 20-byte indexed records. The data are loaded > once and then read from a web app and other client programs. Some of the > queries execute ORDER BY on the results. There are typically less than a > dozen read-only concurrent connections to any one database. I wouldn't count on this being a problem that can be fixed merely by throwing money at it. How many rows does any one of these queries need to access and then ORDER BY? ... > > HP ProLiant DL360p Gen 8 > Dual Intel Xeon 2.4GHz 4-core E5-2609 CPUs > 64GB RAM > 2x146GB 15K SAS hard drives > 3x200GB SATA SLC SSDs > + the usual accessories (optical drive, rail kit, dual power supplies) If your DB is 1G, and will grow to 10G then the IO shouldn't be any problem, as the whole db should be cached in memory. But it can take a surprisingly long time to get it cached in the first place, from a cold start. If that is the problem, pg_prewarm could help. Cheers, Jeff Thank you everyone for your suggestions. It's clear that our current read performance was not limited by hardware. An 'explain analyze' for a sample query is: stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=541002.15..549456.68 rows=3381814 width=20) (actual time=6254.780..7244.074 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..354.385 rows=3381814 loops=1) Total runtime: 7425.854 ms (5 rows) Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? We have an alternate (a bit more conventional) server configuration that we are considering: HP ProLiant DL360p Gen 8 Dual Intel Xeon 3.3GHz 4-core E5-2643 CPUs 128GB PC3-12800 RAM 16x146GB 15K SAS hard drives HP Smart Array P822/2GB FBWC controller + P420i w/ 2GB FBWC + the usual accessories (optical drive, rail kit, dual power supplies) All suggestions welcomed! -Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} The responses: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 4:58:17 PM PDT To: Jeff Janes Cc: "pgsql-performance@postgresql.org" , Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden On May 13, 2013, at 4:24 PM, Jeff Janes wrote: > On Mon, May 13, 2013 at 3:36 PM, Mike McCann wrote: > > Increasing work_mem to 355 MB improves the performance by a factor of 2: > > stoqs_march2013_s=# set work_mem='355MB'; > SET > stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > > I tried changing random_page_cost to from 4 to 1 and saw no change. > > I'm wondering now what changes might get this query to run in less than one second. I think you are worrying about the wrong thing here. What is a web app going to do with 3,381,814 rows, once it obtains them? Your current testing is not testing the time it takes to stream that data to the client, or for the client to do something meaningful with that data. If you only plan to actually fetch a few dozen of those rows, then you probably need to incorporate that into your test, either by using a LIMIT, or by using a mock-up of the actual application to do some timings. Also, what is the type and collation of the column you are sorting on? non-'C' collations of text columns sort about 3 times slower than 'C' collation does. If all the data is in memory, then will faster CPU and memory be the things that help? Yes, those would help (it is not clear to me which of the two would help more), but I think you need to rethink your design of sending the entire database table to the application server for each page-view. Cheers, Jeff Hi Jeff, The datavalue column is double precision: stoqs_march2013_s=# \d+ stoqs_measuredparameter Table "public.stoqs_measuredparameter" Column | Type | Modifiers | Storage | Description ----------------+------------------+----------------------------------------------------------------------+---------+------------- id | integer | not null default nextval('stoqs_measuredparameter_id_seq'::regclass) | plain | measurement_id | integer | not null | plain | parameter_id | integer | not null | plain | datavalue | double precision | not null | plain | Indexes: "stoqs_measuredparameter_pkey" PRIMARY KEY, btree (id) "stoqs_measuredparameter_measurement_id_parameter_id_key" UNIQUE CONSTRAINT, btree (measurement_id, parameter_id) "stoqs_measuredparameter_datavalue" btree (datavalue) "stoqs_measuredparameter_measurement_id" btree (measurement_id) "stoqs_measuredparameter_parameter_id" btree (parameter_id) Foreign-key constraints: "stoqs_measuredparameter_measurement_id_fkey" FOREIGN KEY (measurement_id) REFERENCES stoqs_measurement(id) DEFERRABLE INITIALLY DEFERRED "stoqs_measuredparameter_parameter_id_fkey" FOREIGN KEY (parameter_id) REFERENCES stoqs_parameter(id) DEFERRABLE INITIALLY DEFERRED Has OIDs: no Thanks for the suggestion and advice to examine the web app performance. We've actually taken quite a few steps to optimize how the web app works. The example query I provided is a simple worst-case one that we can use to help us decide on the proper hardware. An actual query performed by the web app is: stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_march2013_s-# stoqs_parameter.name AS parameter__name, stoqs_march2013_s-# stoqs_parameter.standard_name AS parameter__standard_name, stoqs_march2013_s-# stoqs_measurement.depth AS measurement__depth, stoqs_march2013_s-# stoqs_measurement.geom AS measurement__geom, stoqs_march2013_s-# stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_march2013_s-# stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_march2013_s-# stoqs_measuredparameter.datavalue AS datavalue, stoqs_march2013_s-# stoqs_parameter.units AS parameter__units stoqs_march2013_s-# FROM stoqs_parameter p1, stoqs_march2013_s-# stoqs_measuredparameter stoqs_march2013_s-# INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) stoqs_march2013_s-# INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) stoqs_march2013_s-# INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) stoqs_march2013_s-# INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) stoqs_march2013_s-# INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) stoqs_march2013_s-# INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id stoqs_march2013_s-# WHERE (p1.name = 'sea_water_sigma_t') stoqs_march2013_s-# AND (mp1.datavalue > 25.19) stoqs_march2013_s-# AND (mp1.datavalue < 26.01) stoqs_march2013_s-# AND (mp1.parameter_id = p1.id) stoqs_march2013_s-# AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' stoqs_march2013_s(# AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' stoqs_march2013_s(# AND stoqs_parameter.name IN ('fl700_uncorr') stoqs_march2013_s(# AND stoqs_measurement.depth >= -1.88 stoqs_march2013_s(# AND stoqs_platform.name IN ('dorado') stoqs_march2013_s(# AND stoqs_measurement.depth <= 83.57) stoqs_march2013_s-# ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=770.211..770.211 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=770.200..770.200 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=770.036..770.036 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=91.369..680.072 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=91.346..577.633 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=91.280..531.408 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=91.202..116.140 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.002..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=91.194..109.846 rows=20170 loop s=1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=90.794..90.794 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=62.769..62.769 rows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=27.412..2 7.412 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.008..0. 017 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.012..0.012 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.007..0.010 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.004..0.004 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.004..0.004 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.151..0.151 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.035..0.145 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.005..0.069 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.014..0.014 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.008..0.012 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 770.445 ms (42 rows) We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Thanks, Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} Another response: {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 6:09:58 PM PDT To: Mike McCann Cc: Jeff Janes , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden On Mon, May 13, 2013 at 5:58 PM, Mike McCann wrote: > We assume that steps taken to improve the worst-case query scenario will > also improve these kind of queries. If anything above pops out as needing > better planning please let us know that too! Bad assumption. If your real workload will be queries like the one here that takes 700 ms, but you'll be running 10,000 of them a second, you're tuning / hardware choices are going to be much different then if your query is going to be the previous 7 second one. Use realistic queries, not ones that are nothing like what your real ones will be. then use pgbench and its ability to run custom sql scripts to get a REAL idea how your hardware performs. Note that if you will run the slow query you posted like once a minute and roll it up or cache it then don't get too worried about it. Pay attention to the queries that will add up, in aggregate, to your greatest load. {noformat} Not a whole lot of opinions except that tuning a server requires diligent work and performance is something one must do some work to achieve, and cannot be had by simply throwing money at the problem. h4. 20 May 2013 A few more responses over the weekend: {noformat} From: Greg Smith Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 7:44:16 PM PDT To: Mike McCann Cc: pgsql-performance@postgresql.org, Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On 5/13/13 6:36 PM, Mike McCann wrote: > stoqs_march2013_s=# explain analyze select * from > stoqs_measuredparameter order by datavalue; > > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual > time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 > rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > I tried changing random_page_cost to from 4 to 1 and saw no change. Have you tried putting an index by datavalue on this table? Once you've done that, then changing random_page_cost will make using that index look less expensive. Sorting chews through a good bit of CPU time, and that's where all of your runtime is being spent at--once you increase work_mem up very high that is. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? You're trying to fix a fundamental design issue with hardware. That usually doesn't go well. Once you get a box big enough to hold the whole database in RAM, beyond that the differences between server systems are relatively small. -- Greg Smith 2ndQuadrant US greg@2ndQuadrant.com Baltimore, MD PostgreSQL Training, Services, and 24x7 Support www.2ndQuadrant.com {noformat} and {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 8:57:32 PM PDT To: Greg Smith Cc: Mike McCann , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden , Jeff Janes On Sun, May 19, 2013 at 8:44 PM, Greg Smith wrote: >On 5/13/13 6:36 PM, Mike McCann wrote: >> >> stoqs_march2013_s=# explain analyze select * from >> stoqs_measuredparameter order by datavalue; >> >> QUERY PLAN >> >>------------------------------------------------------------------------------------------------------------------------------------------ >> Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual >> time=2503.078..2937.130 rows=3381814 loops=1) >> Sort Key: datavalue >> Sort Method: quicksort Memory: 362509kB >> -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 >> rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) >> Total runtime: 3094.601 ms >> (5 rows) >> >> I tried changing random_page_cost to from 4 to 1 and saw no change. > > > Have you tried putting an index by datavalue on this table? Once you've > done that, then changing random_page_cost will make using that index look > less expensive. Sorting chews through a good bit of CPU time, and that's > where all of your runtime is being spent at--once you increase work_mem up > very high that is. This++ plus cluster on that index if you can. {noformat} ]]> 21364750 1212470 1179702 21397515 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 h4. 16 May 2013 Another post to the pgsql-perform mail list asking for opinions on a more conventional server configuration: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 3:36:03 PM PDT To: pgsql-performance@postgresql.org Cc: Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On May 7, 2013, at 4:21 PM, Jeff Janes wrote: On Thu, May 2, 2013 at 6:35 PM, Scott Marlowe wrote: On Thu, May 2, 2013 at 5:11 PM, Mike McCann wrote: > Hello, > > We are in the fortunate situation of having more money than time to help > solve our PostgreSQL 9.1 performance problem. > > Our server hosts databases that are about 1 GB in size with the largest > tables having order 10 million 20-byte indexed records. The data are loaded > once and then read from a web app and other client programs. Some of the > queries execute ORDER BY on the results. There are typically less than a > dozen read-only concurrent connections to any one database. I wouldn't count on this being a problem that can be fixed merely by throwing money at it. How many rows does any one of these queries need to access and then ORDER BY? ... > > HP ProLiant DL360p Gen 8 > Dual Intel Xeon 2.4GHz 4-core E5-2609 CPUs > 64GB RAM > 2x146GB 15K SAS hard drives > 3x200GB SATA SLC SSDs > + the usual accessories (optical drive, rail kit, dual power supplies) If your DB is 1G, and will grow to 10G then the IO shouldn't be any problem, as the whole db should be cached in memory. But it can take a surprisingly long time to get it cached in the first place, from a cold start. If that is the problem, pg_prewarm could help. Cheers, Jeff Thank you everyone for your suggestions. It's clear that our current read performance was not limited by hardware. An 'explain analyze' for a sample query is: stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=541002.15..549456.68 rows=3381814 width=20) (actual time=6254.780..7244.074 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..354.385 rows=3381814 loops=1) Total runtime: 7425.854 ms (5 rows) Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? We have an alternate (a bit more conventional) server configuration that we are considering: HP ProLiant DL360p Gen 8 Dual Intel Xeon 3.3GHz 4-core E5-2643 CPUs 128GB PC3-12800 RAM 16x146GB 15K SAS hard drives HP Smart Array P822/2GB FBWC controller + P420i w/ 2GB FBWC + the usual accessories (optical drive, rail kit, dual power supplies) All suggestions welcomed! -Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} The responses: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 4:58:17 PM PDT To: Jeff Janes Cc: "pgsql-performance@postgresql.org" , Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden On May 13, 2013, at 4:24 PM, Jeff Janes wrote: > On Mon, May 13, 2013 at 3:36 PM, Mike McCann wrote: > > Increasing work_mem to 355 MB improves the performance by a factor of 2: > > stoqs_march2013_s=# set work_mem='355MB'; > SET > stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > > I tried changing random_page_cost to from 4 to 1 and saw no change. > > I'm wondering now what changes might get this query to run in less than one second. I think you are worrying about the wrong thing here. What is a web app going to do with 3,381,814 rows, once it obtains them? Your current testing is not testing the time it takes to stream that data to the client, or for the client to do something meaningful with that data. If you only plan to actually fetch a few dozen of those rows, then you probably need to incorporate that into your test, either by using a LIMIT, or by using a mock-up of the actual application to do some timings. Also, what is the type and collation of the column you are sorting on? non-'C' collations of text columns sort about 3 times slower than 'C' collation does. If all the data is in memory, then will faster CPU and memory be the things that help? Yes, those would help (it is not clear to me which of the two would help more), but I think you need to rethink your design of sending the entire database table to the application server for each page-view. Cheers, Jeff Hi Jeff, The datavalue column is double precision: stoqs_march2013_s=# \d+ stoqs_measuredparameter Table "public.stoqs_measuredparameter" Column | Type | Modifiers | Storage | Description ----------------+------------------+----------------------------------------------------------------------+---------+------------- id | integer | not null default nextval('stoqs_measuredparameter_id_seq'::regclass) | plain | measurement_id | integer | not null | plain | parameter_id | integer | not null | plain | datavalue | double precision | not null | plain | Indexes: "stoqs_measuredparameter_pkey" PRIMARY KEY, btree (id) "stoqs_measuredparameter_measurement_id_parameter_id_key" UNIQUE CONSTRAINT, btree (measurement_id, parameter_id) "stoqs_measuredparameter_datavalue" btree (datavalue) "stoqs_measuredparameter_measurement_id" btree (measurement_id) "stoqs_measuredparameter_parameter_id" btree (parameter_id) Foreign-key constraints: "stoqs_measuredparameter_measurement_id_fkey" FOREIGN KEY (measurement_id) REFERENCES stoqs_measurement(id) DEFERRABLE INITIALLY DEFERRED "stoqs_measuredparameter_parameter_id_fkey" FOREIGN KEY (parameter_id) REFERENCES stoqs_parameter(id) DEFERRABLE INITIALLY DEFERRED Has OIDs: no Thanks for the suggestion and advice to examine the web app performance. We've actually taken quite a few steps to optimize how the web app works. The example query I provided is a simple worst-case one that we can use to help us decide on the proper hardware. An actual query performed by the web app is: stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_march2013_s-# stoqs_parameter.name AS parameter__name, stoqs_march2013_s-# stoqs_parameter.standard_name AS parameter__standard_name, stoqs_march2013_s-# stoqs_measurement.depth AS measurement__depth, stoqs_march2013_s-# stoqs_measurement.geom AS measurement__geom, stoqs_march2013_s-# stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_march2013_s-# stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_march2013_s-# stoqs_measuredparameter.datavalue AS datavalue, stoqs_march2013_s-# stoqs_parameter.units AS parameter__units stoqs_march2013_s-# FROM stoqs_parameter p1, stoqs_march2013_s-# stoqs_measuredparameter stoqs_march2013_s-# INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) stoqs_march2013_s-# INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) stoqs_march2013_s-# INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) stoqs_march2013_s-# INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) stoqs_march2013_s-# INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) stoqs_march2013_s-# INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id stoqs_march2013_s-# WHERE (p1.name = 'sea_water_sigma_t') stoqs_march2013_s-# AND (mp1.datavalue > 25.19) stoqs_march2013_s-# AND (mp1.datavalue < 26.01) stoqs_march2013_s-# AND (mp1.parameter_id = p1.id) stoqs_march2013_s-# AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' stoqs_march2013_s(# AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' stoqs_march2013_s(# AND stoqs_parameter.name IN ('fl700_uncorr') stoqs_march2013_s(# AND stoqs_measurement.depth >= -1.88 stoqs_march2013_s(# AND stoqs_platform.name IN ('dorado') stoqs_march2013_s(# AND stoqs_measurement.depth <= 83.57) stoqs_march2013_s-# ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=770.211..770.211 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=770.200..770.200 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=770.036..770.036 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=91.369..680.072 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=91.346..577.633 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=91.280..531.408 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=91.202..116.140 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.002..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=91.194..109.846 rows=20170 loop s=1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=90.794..90.794 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=62.769..62.769 rows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=27.412..2 7.412 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.008..0. 017 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.012..0.012 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.007..0.010 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.004..0.004 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.004..0.004 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.151..0.151 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.035..0.145 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.005..0.069 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.014..0.014 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.008..0.012 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 770.445 ms (42 rows) We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Thanks, Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} Another response: {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 6:09:58 PM PDT To: Mike McCann Cc: Jeff Janes , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden On Mon, May 13, 2013 at 5:58 PM, Mike McCann wrote: > We assume that steps taken to improve the worst-case query scenario will > also improve these kind of queries. If anything above pops out as needing > better planning please let us know that too! Bad assumption. If your real workload will be queries like the one here that takes 700 ms, but you'll be running 10,000 of them a second, you're tuning / hardware choices are going to be much different then if your query is going to be the previous 7 second one. Use realistic queries, not ones that are nothing like what your real ones will be. then use pgbench and its ability to run custom sql scripts to get a REAL idea how your hardware performs. Note that if you will run the slow query you posted like once a minute and roll it up or cache it then don't get too worried about it. Pay attention to the queries that will add up, in aggregate, to your greatest load. {noformat} Not a whole lot of opinions except that tuning a server requires diligent work and performance is something one must do some work to achieve, and cannot be had by simply throwing money at the problem. ]]> 21364747 1212467 1179699 21397516 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 h4. 16 May 2013 Another post to the pgsql-perform mail list asking for opinions on a more conventional server configuration: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 3:36:03 PM PDT To: pgsql-performance@postgresql.org Cc: Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On May 7, 2013, at 4:21 PM, Jeff Janes wrote: On Thu, May 2, 2013 at 6:35 PM, Scott Marlowe wrote: On Thu, May 2, 2013 at 5:11 PM, Mike McCann wrote: > Hello, > > We are in the fortunate situation of having more money than time to help > solve our PostgreSQL 9.1 performance problem. > > Our server hosts databases that are about 1 GB in size with the largest > tables having order 10 million 20-byte indexed records. The data are loaded > once and then read from a web app and other client programs. Some of the > queries execute ORDER BY on the results. There are typically less than a > dozen read-only concurrent connections to any one database. I wouldn't count on this being a problem that can be fixed merely by throwing money at it. How many rows does any one of these queries need to access and then ORDER BY? ... > > HP ProLiant DL360p Gen 8 > Dual Intel Xeon 2.4GHz 4-core E5-2609 CPUs > 64GB RAM > 2x146GB 15K SAS hard drives > 3x200GB SATA SLC SSDs > + the usual accessories (optical drive, rail kit, dual power supplies) If your DB is 1G, and will grow to 10G then the IO shouldn't be any problem, as the whole db should be cached in memory. But it can take a surprisingly long time to get it cached in the first place, from a cold start. If that is the problem, pg_prewarm could help. Cheers, Jeff Thank you everyone for your suggestions. It's clear that our current read performance was not limited by hardware. An 'explain analyze' for a sample query is: stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=541002.15..549456.68 rows=3381814 width=20) (actual time=6254.780..7244.074 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..354.385 rows=3381814 loops=1) Total runtime: 7425.854 ms (5 rows) Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? We have an alternate (a bit more conventional) server configuration that we are considering: HP ProLiant DL360p Gen 8 Dual Intel Xeon 3.3GHz 4-core E5-2643 CPUs 128GB PC3-12800 RAM 16x146GB 15K SAS hard drives HP Smart Array P822/2GB FBWC controller + P420i w/ 2GB FBWC + the usual accessories (optical drive, rail kit, dual power supplies) All suggestions welcomed! -Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} The responses: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 4:58:17 PM PDT To: Jeff Janes Cc: "pgsql-performance@postgresql.org" , Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden On May 13, 2013, at 4:24 PM, Jeff Janes wrote: > On Mon, May 13, 2013 at 3:36 PM, Mike McCann wrote: > > Increasing work_mem to 355 MB improves the performance by a factor of 2: > > stoqs_march2013_s=# set work_mem='355MB'; > SET > stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > > I tried changing random_page_cost to from 4 to 1 and saw no change. > > I'm wondering now what changes might get this query to run in less than one second. I think you are worrying about the wrong thing here. What is a web app going to do with 3,381,814 rows, once it obtains them? Your current testing is not testing the time it takes to stream that data to the client, or for the client to do something meaningful with that data. If you only plan to actually fetch a few dozen of those rows, then you probably need to incorporate that into your test, either by using a LIMIT, or by using a mock-up of the actual application to do some timings. Also, what is the type and collation of the column you are sorting on? non-'C' collations of text columns sort about 3 times slower than 'C' collation does. If all the data is in memory, then will faster CPU and memory be the things that help? Yes, those would help (it is not clear to me which of the two would help more), but I think you need to rethink your design of sending the entire database table to the application server for each page-view. Cheers, Jeff Hi Jeff, The datavalue column is double precision: stoqs_march2013_s=# \d+ stoqs_measuredparameter Table "public.stoqs_measuredparameter" Column | Type | Modifiers | Storage | Description ----------------+------------------+----------------------------------------------------------------------+---------+------------- id | integer | not null default nextval('stoqs_measuredparameter_id_seq'::regclass) | plain | measurement_id | integer | not null | plain | parameter_id | integer | not null | plain | datavalue | double precision | not null | plain | Indexes: "stoqs_measuredparameter_pkey" PRIMARY KEY, btree (id) "stoqs_measuredparameter_measurement_id_parameter_id_key" UNIQUE CONSTRAINT, btree (measurement_id, parameter_id) "stoqs_measuredparameter_datavalue" btree (datavalue) "stoqs_measuredparameter_measurement_id" btree (measurement_id) "stoqs_measuredparameter_parameter_id" btree (parameter_id) Foreign-key constraints: "stoqs_measuredparameter_measurement_id_fkey" FOREIGN KEY (measurement_id) REFERENCES stoqs_measurement(id) DEFERRABLE INITIALLY DEFERRED "stoqs_measuredparameter_parameter_id_fkey" FOREIGN KEY (parameter_id) REFERENCES stoqs_parameter(id) DEFERRABLE INITIALLY DEFERRED Has OIDs: no Thanks for the suggestion and advice to examine the web app performance. We've actually taken quite a few steps to optimize how the web app works. The example query I provided is a simple worst-case one that we can use to help us decide on the proper hardware. An actual query performed by the web app is: stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_march2013_s-# stoqs_parameter.name AS parameter__name, stoqs_march2013_s-# stoqs_parameter.standard_name AS parameter__standard_name, stoqs_march2013_s-# stoqs_measurement.depth AS measurement__depth, stoqs_march2013_s-# stoqs_measurement.geom AS measurement__geom, stoqs_march2013_s-# stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_march2013_s-# stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_march2013_s-# stoqs_measuredparameter.datavalue AS datavalue, stoqs_march2013_s-# stoqs_parameter.units AS parameter__units stoqs_march2013_s-# FROM stoqs_parameter p1, stoqs_march2013_s-# stoqs_measuredparameter stoqs_march2013_s-# INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) stoqs_march2013_s-# INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) stoqs_march2013_s-# INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) stoqs_march2013_s-# INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) stoqs_march2013_s-# INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) stoqs_march2013_s-# INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id stoqs_march2013_s-# WHERE (p1.name = 'sea_water_sigma_t') stoqs_march2013_s-# AND (mp1.datavalue > 25.19) stoqs_march2013_s-# AND (mp1.datavalue < 26.01) stoqs_march2013_s-# AND (mp1.parameter_id = p1.id) stoqs_march2013_s-# AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' stoqs_march2013_s(# AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' stoqs_march2013_s(# AND stoqs_parameter.name IN ('fl700_uncorr') stoqs_march2013_s(# AND stoqs_measurement.depth >= -1.88 stoqs_march2013_s(# AND stoqs_platform.name IN ('dorado') stoqs_march2013_s(# AND stoqs_measurement.depth <= 83.57) stoqs_march2013_s-# ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=770.211..770.211 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=770.200..770.200 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=770.036..770.036 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=91.369..680.072 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=91.346..577.633 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=91.280..531.408 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=91.202..116.140 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.002..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=91.194..109.846 rows=20170 loop s=1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=90.794..90.794 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=62.769..62.769 rows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=27.412..2 7.412 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.008..0. 017 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.012..0.012 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.007..0.010 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.004..0.004 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.004..0.004 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.151..0.151 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.035..0.145 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.005..0.069 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.014..0.014 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.008..0.012 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 770.445 ms (42 rows) We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Thanks, Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} Another response: {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 6:09:58 PM PDT To: Mike McCann Cc: Jeff Janes , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden On Mon, May 13, 2013 at 5:58 PM, Mike McCann wrote: > We assume that steps taken to improve the worst-case query scenario will > also improve these kind of queries. If anything above pops out as needing > better planning please let us know that too! Bad assumption. If your real workload will be queries like the one here that takes 700 ms, but you'll be running 10,000 of them a second, you're tuning / hardware choices are going to be much different then if your query is going to be the previous 7 second one. Use realistic queries, not ones that are nothing like what your real ones will be. then use pgbench and its ability to run custom sql scripts to get a REAL idea how your hardware performs. Note that if you will run the slow query you posted like once a minute and roll it up or cache it then don't get too worried about it. Pay attention to the queries that will add up, in aggregate, to your greatest load. {noformat} Not a whole lot of opinions except that tuning a server requires diligent work and performance is something one must do some work to achieve, and cannot be had by simply throwing money at the problem. h4. 20 May 2013 A few more responses over the weekend: {noformat} From: Greg Smith Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 7:44:16 PM PDT To: Mike McCann Cc: pgsql-performance@postgresql.org, Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On 5/13/13 6:36 PM, Mike McCann wrote: > stoqs_march2013_s=# explain analyze select * from > stoqs_measuredparameter order by datavalue; > > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual > time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 > rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > I tried changing random_page_cost to from 4 to 1 and saw no change. Have you tried putting an index by datavalue on this table? Once you've done that, then changing random_page_cost will make using that index look less expensive. Sorting chews through a good bit of CPU time, and that's where all of your runtime is being spent at--once you increase work_mem up very high that is. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? You're trying to fix a fundamental design issue with hardware. That usually doesn't go well. Once you get a box big enough to hold the whole database in RAM, beyond that the differences between server systems are relatively small. -- Greg Smith 2ndQuadrant US greg@2ndQuadrant.com Baltimore, MD PostgreSQL Training, Services, and 24x7 Support www.2ndQuadrant.com {noformat} and {nofromat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 8:57:32 PM PDT To: Greg Smith Cc: Mike McCann , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden , Jeff Janes On Sun, May 19, 2013 at 8:44 PM, Greg Smith wrote: >On 5/13/13 6:36 PM, Mike McCann wrote: >> >> stoqs_march2013_s=# explain analyze select * from >> stoqs_measuredparameter order by datavalue; >> >> QUERY PLAN >> >>------------------------------------------------------------------------------------------------------------------------------------------ >> Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual >> time=2503.078..2937.130 rows=3381814 loops=1) >> Sort Key: datavalue >> Sort Method: quicksort Memory: 362509kB >> -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 >> rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) >> Total runtime: 3094.601 ms >> (5 rows) >> >> I tried changing random_page_cost to from 4 to 1 and saw no change. > > > Have you tried putting an index by datavalue on this table? Once you've > done that, then changing random_page_cost will make using that index look > less expensive. Sorting chews through a good bit of CPU time, and that's > where all of your runtime is being spent at--once you increase work_mem up > very high that is. This++ plus cluster on that index if you can. {noformat} ]]> 21364748 8945987 8913234 1212468 1179700 8945994 8913241 1212471 1179703 1212472 1179704 21397509 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 h4. 16 May 2013 Another post to the pgsql-perform mail list: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 3:36:03 PM PDT To: pgsql-performance@postgresql.org Cc: Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On May 7, 2013, at 4:21 PM, Jeff Janes wrote: On Thu, May 2, 2013 at 6:35 PM, Scott Marlowe wrote: On Thu, May 2, 2013 at 5:11 PM, Mike McCann wrote: > Hello, > > We are in the fortunate situation of having more money than time to help > solve our PostgreSQL 9.1 performance problem. > > Our server hosts databases that are about 1 GB in size with the largest > tables having order 10 million 20-byte indexed records. The data are loaded > once and then read from a web app and other client programs. Some of the > queries execute ORDER BY on the results. There are typically less than a > dozen read-only concurrent connections to any one database. I wouldn't count on this being a problem that can be fixed merely by throwing money at it. How many rows does any one of these queries need to access and then ORDER BY? ... > > HP ProLiant DL360p Gen 8 > Dual Intel Xeon 2.4GHz 4-core E5-2609 CPUs > 64GB RAM > 2x146GB 15K SAS hard drives > 3x200GB SATA SLC SSDs > + the usual accessories (optical drive, rail kit, dual power supplies) If your DB is 1G, and will grow to 10G then the IO shouldn't be any problem, as the whole db should be cached in memory. But it can take a surprisingly long time to get it cached in the first place, from a cold start. If that is the problem, pg_prewarm could help. Cheers, Jeff Thank you everyone for your suggestions. It's clear that our current read performance was not limited by hardware. An 'explain analyze' for a sample query is: stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=541002.15..549456.68 rows=3381814 width=20) (actual time=6254.780..7244.074 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..354.385 rows=3381814 loops=1) Total runtime: 7425.854 ms (5 rows) Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? We have an alternate (a bit more conventional) server configuration that we are considering: HP ProLiant DL360p Gen 8 Dual Intel Xeon 3.3GHz 4-core E5-2643 CPUs 128GB PC3-12800 RAM 16x146GB 15K SAS hard drives HP Smart Array P822/2GB FBWC controller + P420i w/ 2GB FBWC + the usual accessories (optical drive, rail kit, dual power supplies) All suggestions welcomed! -Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} The responses: {quote} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 4:58:17 PM PDT To: Jeff Janes Cc: "pgsql-performance@postgresql.org" , Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden On May 13, 2013, at 4:24 PM, Jeff Janes wrote: On Mon, May 13, 2013 at 3:36 PM, Mike McCann wrote: Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. I think you are worrying about the wrong thing here. What is a web app going to do with 3,381,814 rows, once it obtains them? Your current testing is not testing the time it takes to stream that data to the client, or for the client to do something meaningful with that data. If you only plan to actually fetch a few dozen of those rows, then you probably need to incorporate that into your test, either by using a LIMIT, or by using a mock-up of the actual application to do some timings. Also, what is the type and collation of the column you are sorting on? non-'C' collations of text columns sort about 3 times slower than 'C' collation does. If all the data is in memory, then will faster CPU and memory be the things that help? Yes, those would help (it is not clear to me which of the two would help more), but I think you need to rethink your design of sending the entire database table to the application server for each page-view. Cheers, Jeff Hi Jeff, The datavalue column is double precision: stoqs_march2013_s=# \d+ stoqs_measuredparameter Table "public.stoqs_measuredparameter" Column | Type | Modifiers | Storage | Description ----------------+------------------+----------------------------------------------------------------------+---------+------------- id | integer | not null default nextval('stoqs_measuredparameter_id_seq'::regclass) | plain | measurement_id | integer | not null | plain | parameter_id | integer | not null | plain | datavalue | double precision | not null | plain | Indexes: "stoqs_measuredparameter_pkey" PRIMARY KEY, btree (id) "stoqs_measuredparameter_measurement_id_parameter_id_key" UNIQUE CONSTRAINT, btree (measurement_id, parameter_id) "stoqs_measuredparameter_datavalue" btree (datavalue) "stoqs_measuredparameter_measurement_id" btree (measurement_id) "stoqs_measuredparameter_parameter_id" btree (parameter_id) Foreign-key constraints: "stoqs_measuredparameter_measurement_id_fkey" FOREIGN KEY (measurement_id) REFERENCES stoqs_measurement(id) DEFERRABLE INITIALLY DEFERRED "stoqs_measuredparameter_parameter_id_fkey" FOREIGN KEY (parameter_id) REFERENCES stoqs_parameter(id) DEFERRABLE INITIALLY DEFERRED Has OIDs: no Thanks for the suggestion and advice to examine the web app performance. We've actually taken quite a few steps to optimize how the web app works. The example query I provided is a simple worst-case one that we can use to help us decide on the proper hardware. An actual query performed by the web app is: stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_march2013_s-# stoqs_parameter.name AS parameter__name, stoqs_march2013_s-# stoqs_parameter.standard_name AS parameter__standard_name, stoqs_march2013_s-# stoqs_measurement.depth AS measurement__depth, stoqs_march2013_s-# stoqs_measurement.geom AS measurement__geom, stoqs_march2013_s-# stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_march2013_s-# stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_march2013_s-# stoqs_measuredparameter.datavalue AS datavalue, stoqs_march2013_s-# stoqs_parameter.units AS parameter__units stoqs_march2013_s-# FROM stoqs_parameter p1, stoqs_march2013_s-# stoqs_measuredparameter stoqs_march2013_s-# INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) stoqs_march2013_s-# INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) stoqs_march2013_s-# INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) stoqs_march2013_s-# INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) stoqs_march2013_s-# INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) stoqs_march2013_s-# INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id stoqs_march2013_s-# WHERE (p1.name = 'sea_water_sigma_t') stoqs_march2013_s-# AND (mp1.datavalue > 25.19) stoqs_march2013_s-# AND (mp1.datavalue < 26.01) stoqs_march2013_s-# AND (mp1.parameter_id = p1.id) stoqs_march2013_s-# AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' stoqs_march2013_s(# AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' stoqs_march2013_s(# AND stoqs_parameter.name IN ('fl700_uncorr') stoqs_march2013_s(# AND stoqs_measurement.depth >= -1.88 stoqs_march2013_s(# AND stoqs_platform.name IN ('dorado') stoqs_march2013_s(# AND stoqs_measurement.depth <= 83.57) stoqs_march2013_s-# ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=770.211..770.211 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=770.200..770.200 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=770.036..770.036 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=91.369..680.072 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=91.346..577.633 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=91.280..531.408 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=91.202..116.140 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.002..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=91.194..109.846 rows=20170 loop s=1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=90.794..90.794 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=62.769..62.769 rows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=27.412..2 7.412 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.008..0. 017 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.012..0.012 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.007..0.010 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.004..0.004 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.004..0.004 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.151..0.151 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.035..0.145 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.005..0.069 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.014..0.014 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.008..0.012 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 770.445 ms (42 rows) We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Thanks, Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {quote} {quote} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 6:09:58 PM PDT To: Mike McCann Cc: Jeff Janes , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden On Mon, May 13, 2013 at 5:58 PM, Mike McCann wrote: We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Bad assumption. If your real workload will be queries like the one here that takes 700 ms, but you'll be running 10,000 of them a second, you're tuning / hardware choices are going to be much different then if your query is going to be the previous 7 second one. Use realistic queries, not ones that are nothing like what your real ones will be. then use pgbench and its ability to run custom sql scripts to get a REAL idea how your hardware performs. Note that if you will run the slow query you posted like once a minute and roll it up or cache it then don't get too worried about it. Pay attention to the queries that will add up, in aggregate, to your greatest load. {quote} ]]> 21364741 21397507 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 h4. 16 May 2013 Another post to the pgsql-perform mail list: {quote} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 3:36:03 PM PDT To: pgsql-performance@postgresql.org Cc: Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On May 7, 2013, at 4:21 PM, Jeff Janes wrote: On Thu, May 2, 2013 at 6:35 PM, Scott Marlowe wrote: On Thu, May 2, 2013 at 5:11 PM, Mike McCann wrote: > Hello, > > We are in the fortunate situation of having more money than time to help > solve our PostgreSQL 9.1 performance problem. > > Our server hosts databases that are about 1 GB in size with the largest > tables having order 10 million 20-byte indexed records. The data are loaded > once and then read from a web app and other client programs. Some of the > queries execute ORDER BY on the results. There are typically less than a > dozen read-only concurrent connections to any one database. I wouldn't count on this being a problem that can be fixed merely by throwing money at it. How many rows does any one of these queries need to access and then ORDER BY? ... > > HP ProLiant DL360p Gen 8 > Dual Intel Xeon 2.4GHz 4-core E5-2609 CPUs > 64GB RAM > 2x146GB 15K SAS hard drives > 3x200GB SATA SLC SSDs > + the usual accessories (optical drive, rail kit, dual power supplies) If your DB is 1G, and will grow to 10G then the IO shouldn't be any problem, as the whole db should be cached in memory. But it can take a surprisingly long time to get it cached in the first place, from a cold start. If that is the problem, pg_prewarm could help. Cheers, Jeff Thank you everyone for your suggestions. It's clear that our current read performance was not limited by hardware. An 'explain analyze' for a sample query is: stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=541002.15..549456.68 rows=3381814 width=20) (actual time=6254.780..7244.074 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..354.385 rows=3381814 loops=1) Total runtime: 7425.854 ms (5 rows) Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? We have an alternate (a bit more conventional) server configuration that we are considering: HP ProLiant DL360p Gen 8 Dual Intel Xeon 3.3GHz 4-core E5-2643 CPUs 128GB PC3-12800 RAM 16x146GB 15K SAS hard drives HP Smart Array P822/2GB FBWC controller + P420i w/ 2GB FBWC + the usual accessories (optical drive, rail kit, dual power supplies) All suggestions welcomed! -Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {quote} The responses: {quote} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 4:58:17 PM PDT To: Jeff Janes Cc: "pgsql-performance@postgresql.org" , Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden On May 13, 2013, at 4:24 PM, Jeff Janes wrote: On Mon, May 13, 2013 at 3:36 PM, Mike McCann wrote: Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. I think you are worrying about the wrong thing here. What is a web app going to do with 3,381,814 rows, once it obtains them? Your current testing is not testing the time it takes to stream that data to the client, or for the client to do something meaningful with that data. If you only plan to actually fetch a few dozen of those rows, then you probably need to incorporate that into your test, either by using a LIMIT, or by using a mock-up of the actual application to do some timings. Also, what is the type and collation of the column you are sorting on? non-'C' collations of text columns sort about 3 times slower than 'C' collation does. If all the data is in memory, then will faster CPU and memory be the things that help? Yes, those would help (it is not clear to me which of the two would help more), but I think you need to rethink your design of sending the entire database table to the application server for each page-view. Cheers, Jeff Hi Jeff, The datavalue column is double precision: stoqs_march2013_s=# \d+ stoqs_measuredparameter Table "public.stoqs_measuredparameter" Column | Type | Modifiers | Storage | Description ----------------+------------------+----------------------------------------------------------------------+---------+------------- id | integer | not null default nextval('stoqs_measuredparameter_id_seq'::regclass) | plain | measurement_id | integer | not null | plain | parameter_id | integer | not null | plain | datavalue | double precision | not null | plain | Indexes: "stoqs_measuredparameter_pkey" PRIMARY KEY, btree (id) "stoqs_measuredparameter_measurement_id_parameter_id_key" UNIQUE CONSTRAINT, btree (measurement_id, parameter_id) "stoqs_measuredparameter_datavalue" btree (datavalue) "stoqs_measuredparameter_measurement_id" btree (measurement_id) "stoqs_measuredparameter_parameter_id" btree (parameter_id) Foreign-key constraints: "stoqs_measuredparameter_measurement_id_fkey" FOREIGN KEY (measurement_id) REFERENCES stoqs_measurement(id) DEFERRABLE INITIALLY DEFERRED "stoqs_measuredparameter_parameter_id_fkey" FOREIGN KEY (parameter_id) REFERENCES stoqs_parameter(id) DEFERRABLE INITIALLY DEFERRED Has OIDs: no Thanks for the suggestion and advice to examine the web app performance. We've actually taken quite a few steps to optimize how the web app works. The example query I provided is a simple worst-case one that we can use to help us decide on the proper hardware. An actual query performed by the web app is: stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_march2013_s-# stoqs_parameter.name AS parameter__name, stoqs_march2013_s-# stoqs_parameter.standard_name AS parameter__standard_name, stoqs_march2013_s-# stoqs_measurement.depth AS measurement__depth, stoqs_march2013_s-# stoqs_measurement.geom AS measurement__geom, stoqs_march2013_s-# stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_march2013_s-# stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_march2013_s-# stoqs_measuredparameter.datavalue AS datavalue, stoqs_march2013_s-# stoqs_parameter.units AS parameter__units stoqs_march2013_s-# FROM stoqs_parameter p1, stoqs_march2013_s-# stoqs_measuredparameter stoqs_march2013_s-# INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) stoqs_march2013_s-# INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) stoqs_march2013_s-# INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) stoqs_march2013_s-# INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) stoqs_march2013_s-# INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) stoqs_march2013_s-# INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id stoqs_march2013_s-# WHERE (p1.name = 'sea_water_sigma_t') stoqs_march2013_s-# AND (mp1.datavalue > 25.19) stoqs_march2013_s-# AND (mp1.datavalue < 26.01) stoqs_march2013_s-# AND (mp1.parameter_id = p1.id) stoqs_march2013_s-# AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' stoqs_march2013_s(# AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' stoqs_march2013_s(# AND stoqs_parameter.name IN ('fl700_uncorr') stoqs_march2013_s(# AND stoqs_measurement.depth >= -1.88 stoqs_march2013_s(# AND stoqs_platform.name IN ('dorado') stoqs_march2013_s(# AND stoqs_measurement.depth <= 83.57) stoqs_march2013_s-# ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=770.211..770.211 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=770.200..770.200 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=770.036..770.036 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=91.369..680.072 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=91.346..577.633 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=91.280..531.408 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=91.202..116.140 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.002..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=91.194..109.846 rows=20170 loop s=1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=90.794..90.794 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=62.769..62.769 rows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=27.412..2 7.412 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.008..0. 017 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.012..0.012 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.007..0.010 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.004..0.004 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.004..0.004 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.151..0.151 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.035..0.145 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.005..0.069 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.014..0.014 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.008..0.012 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 770.445 ms (42 rows) We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Thanks, Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {quote} {quote} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 6:09:58 PM PDT To: Mike McCann Cc: Jeff Janes , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden On Mon, May 13, 2013 at 5:58 PM, Mike McCann wrote: We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Bad assumption. If your real workload will be queries like the one here that takes 700 ms, but you'll be running 10,000 of them a second, you're tuning / hardware choices are going to be much different then if your query is going to be the previous 7 second one. Use realistic queries, not ones that are nothing like what your real ones will be. then use pgbench and its ability to run custom sql scripts to get a REAL idea how your hardware performs. Note that if you will run the slow query you posted like once a minute and roll it up or cache it then don't get too worried about it. Pay attention to the queries that will add up, in aggregate, to your greatest load. {quote} ]]> 21364739 21397513 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 h4. 16 May 2013 Another post to the pgsql-perform mail list asking for opinions on a more conventional server configuration: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 3:36:03 PM PDT To: pgsql-performance@postgresql.org Cc: Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On May 7, 2013, at 4:21 PM, Jeff Janes wrote: On Thu, May 2, 2013 at 6:35 PM, Scott Marlowe wrote: On Thu, May 2, 2013 at 5:11 PM, Mike McCann wrote: > Hello, > > We are in the fortunate situation of having more money than time to help > solve our PostgreSQL 9.1 performance problem. > > Our server hosts databases that are about 1 GB in size with the largest > tables having order 10 million 20-byte indexed records. The data are loaded > once and then read from a web app and other client programs. Some of the > queries execute ORDER BY on the results. There are typically less than a > dozen read-only concurrent connections to any one database. I wouldn't count on this being a problem that can be fixed merely by throwing money at it. How many rows does any one of these queries need to access and then ORDER BY? ... > > HP ProLiant DL360p Gen 8 > Dual Intel Xeon 2.4GHz 4-core E5-2609 CPUs > 64GB RAM > 2x146GB 15K SAS hard drives > 3x200GB SATA SLC SSDs > + the usual accessories (optical drive, rail kit, dual power supplies) If your DB is 1G, and will grow to 10G then the IO shouldn't be any problem, as the whole db should be cached in memory. But it can take a surprisingly long time to get it cached in the first place, from a cold start. If that is the problem, pg_prewarm could help. Cheers, Jeff Thank you everyone for your suggestions. It's clear that our current read performance was not limited by hardware. An 'explain analyze' for a sample query is: stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=541002.15..549456.68 rows=3381814 width=20) (actual time=6254.780..7244.074 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..354.385 rows=3381814 loops=1) Total runtime: 7425.854 ms (5 rows) Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? We have an alternate (a bit more conventional) server configuration that we are considering: HP ProLiant DL360p Gen 8 Dual Intel Xeon 3.3GHz 4-core E5-2643 CPUs 128GB PC3-12800 RAM 16x146GB 15K SAS hard drives HP Smart Array P822/2GB FBWC controller + P420i w/ 2GB FBWC + the usual accessories (optical drive, rail kit, dual power supplies) All suggestions welcomed! -Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} The responses: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 4:58:17 PM PDT To: Jeff Janes Cc: "pgsql-performance@postgresql.org" , Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden On May 13, 2013, at 4:24 PM, Jeff Janes wrote: On Mon, May 13, 2013 at 3:36 PM, Mike McCann wrote: Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. I think you are worrying about the wrong thing here. What is a web app going to do with 3,381,814 rows, once it obtains them? Your current testing is not testing the time it takes to stream that data to the client, or for the client to do something meaningful with that data. If you only plan to actually fetch a few dozen of those rows, then you probably need to incorporate that into your test, either by using a LIMIT, or by using a mock-up of the actual application to do some timings. Also, what is the type and collation of the column you are sorting on? non-'C' collations of text columns sort about 3 times slower than 'C' collation does. If all the data is in memory, then will faster CPU and memory be the things that help? Yes, those would help (it is not clear to me which of the two would help more), but I think you need to rethink your design of sending the entire database table to the application server for each page-view. Cheers, Jeff Hi Jeff, The datavalue column is double precision: stoqs_march2013_s=# \d+ stoqs_measuredparameter Table "public.stoqs_measuredparameter" Column | Type | Modifiers | Storage | Description ----------------+------------------+----------------------------------------------------------------------+---------+------------- id | integer | not null default nextval('stoqs_measuredparameter_id_seq'::regclass) | plain | measurement_id | integer | not null | plain | parameter_id | integer | not null | plain | datavalue | double precision | not null | plain | Indexes: "stoqs_measuredparameter_pkey" PRIMARY KEY, btree (id) "stoqs_measuredparameter_measurement_id_parameter_id_key" UNIQUE CONSTRAINT, btree (measurement_id, parameter_id) "stoqs_measuredparameter_datavalue" btree (datavalue) "stoqs_measuredparameter_measurement_id" btree (measurement_id) "stoqs_measuredparameter_parameter_id" btree (parameter_id) Foreign-key constraints: "stoqs_measuredparameter_measurement_id_fkey" FOREIGN KEY (measurement_id) REFERENCES stoqs_measurement(id) DEFERRABLE INITIALLY DEFERRED "stoqs_measuredparameter_parameter_id_fkey" FOREIGN KEY (parameter_id) REFERENCES stoqs_parameter(id) DEFERRABLE INITIALLY DEFERRED Has OIDs: no Thanks for the suggestion and advice to examine the web app performance. We've actually taken quite a few steps to optimize how the web app works. The example query I provided is a simple worst-case one that we can use to help us decide on the proper hardware. An actual query performed by the web app is: stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_march2013_s-# stoqs_parameter.name AS parameter__name, stoqs_march2013_s-# stoqs_parameter.standard_name AS parameter__standard_name, stoqs_march2013_s-# stoqs_measurement.depth AS measurement__depth, stoqs_march2013_s-# stoqs_measurement.geom AS measurement__geom, stoqs_march2013_s-# stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_march2013_s-# stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_march2013_s-# stoqs_measuredparameter.datavalue AS datavalue, stoqs_march2013_s-# stoqs_parameter.units AS parameter__units stoqs_march2013_s-# FROM stoqs_parameter p1, stoqs_march2013_s-# stoqs_measuredparameter stoqs_march2013_s-# INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) stoqs_march2013_s-# INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) stoqs_march2013_s-# INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) stoqs_march2013_s-# INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) stoqs_march2013_s-# INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) stoqs_march2013_s-# INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id stoqs_march2013_s-# WHERE (p1.name = 'sea_water_sigma_t') stoqs_march2013_s-# AND (mp1.datavalue > 25.19) stoqs_march2013_s-# AND (mp1.datavalue < 26.01) stoqs_march2013_s-# AND (mp1.parameter_id = p1.id) stoqs_march2013_s-# AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' stoqs_march2013_s(# AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' stoqs_march2013_s(# AND stoqs_parameter.name IN ('fl700_uncorr') stoqs_march2013_s(# AND stoqs_measurement.depth >= -1.88 stoqs_march2013_s(# AND stoqs_platform.name IN ('dorado') stoqs_march2013_s(# AND stoqs_measurement.depth <= 83.57) stoqs_march2013_s-# ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=770.211..770.211 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=770.200..770.200 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=770.036..770.036 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=91.369..680.072 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=91.346..577.633 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=91.280..531.408 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=91.202..116.140 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.002..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=91.194..109.846 rows=20170 loop s=1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=90.794..90.794 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=62.769..62.769 rows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=27.412..2 7.412 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.008..0. 017 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.012..0.012 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.007..0.010 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.004..0.004 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.004..0.004 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.151..0.151 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.035..0.145 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.005..0.069 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.014..0.014 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.008..0.012 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 770.445 ms (42 rows) We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Thanks, Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} Another response: {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 6:09:58 PM PDT To: Mike McCann Cc: Jeff Janes , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden On Mon, May 13, 2013 at 5:58 PM, Mike McCann wrote: We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Bad assumption. If your real workload will be queries like the one here that takes 700 ms, but you'll be running 10,000 of them a second, you're tuning / hardware choices are going to be much different then if your query is going to be the previous 7 second one. Use realistic queries, not ones that are nothing like what your real ones will be. then use pgbench and its ability to run custom sql scripts to get a REAL idea how your hardware performs. Note that if you will run the slow query you posted like once a minute and roll it up or cache it then don't get too worried about it. Pay attention to the queries that will add up, in aggregate, to your greatest load. {noformat} Not a whole lot of opinions except that tuning a server requires diligent work and performance is something one must do some work to achieve, and cannot be had by simply throwing money at the problem. ]]> 21364745 21397511 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 h4. 16 May 2013 Another post to the pgsql-perform mail list: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 3:36:03 PM PDT To: pgsql-performance@postgresql.org Cc: Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On May 7, 2013, at 4:21 PM, Jeff Janes wrote: On Thu, May 2, 2013 at 6:35 PM, Scott Marlowe wrote: On Thu, May 2, 2013 at 5:11 PM, Mike McCann wrote: > Hello, > > We are in the fortunate situation of having more money than time to help > solve our PostgreSQL 9.1 performance problem. > > Our server hosts databases that are about 1 GB in size with the largest > tables having order 10 million 20-byte indexed records. The data are loaded > once and then read from a web app and other client programs. Some of the > queries execute ORDER BY on the results. There are typically less than a > dozen read-only concurrent connections to any one database. I wouldn't count on this being a problem that can be fixed merely by throwing money at it. How many rows does any one of these queries need to access and then ORDER BY? ... > > HP ProLiant DL360p Gen 8 > Dual Intel Xeon 2.4GHz 4-core E5-2609 CPUs > 64GB RAM > 2x146GB 15K SAS hard drives > 3x200GB SATA SLC SSDs > + the usual accessories (optical drive, rail kit, dual power supplies) If your DB is 1G, and will grow to 10G then the IO shouldn't be any problem, as the whole db should be cached in memory. But it can take a surprisingly long time to get it cached in the first place, from a cold start. If that is the problem, pg_prewarm could help. Cheers, Jeff Thank you everyone for your suggestions. It's clear that our current read performance was not limited by hardware. An 'explain analyze' for a sample query is: stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=541002.15..549456.68 rows=3381814 width=20) (actual time=6254.780..7244.074 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..354.385 rows=3381814 loops=1) Total runtime: 7425.854 ms (5 rows) Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? We have an alternate (a bit more conventional) server configuration that we are considering: HP ProLiant DL360p Gen 8 Dual Intel Xeon 3.3GHz 4-core E5-2643 CPUs 128GB PC3-12800 RAM 16x146GB 15K SAS hard drives HP Smart Array P822/2GB FBWC controller + P420i w/ 2GB FBWC + the usual accessories (optical drive, rail kit, dual power supplies) All suggestions welcomed! -Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} The responses: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 4:58:17 PM PDT To: Jeff Janes Cc: "pgsql-performance@postgresql.org" , Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden On May 13, 2013, at 4:24 PM, Jeff Janes wrote: On Mon, May 13, 2013 at 3:36 PM, Mike McCann wrote: Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. I think you are worrying about the wrong thing here. What is a web app going to do with 3,381,814 rows, once it obtains them? Your current testing is not testing the time it takes to stream that data to the client, or for the client to do something meaningful with that data. If you only plan to actually fetch a few dozen of those rows, then you probably need to incorporate that into your test, either by using a LIMIT, or by using a mock-up of the actual application to do some timings. Also, what is the type and collation of the column you are sorting on? non-'C' collations of text columns sort about 3 times slower than 'C' collation does. If all the data is in memory, then will faster CPU and memory be the things that help? Yes, those would help (it is not clear to me which of the two would help more), but I think you need to rethink your design of sending the entire database table to the application server for each page-view. Cheers, Jeff Hi Jeff, The datavalue column is double precision: stoqs_march2013_s=# \d+ stoqs_measuredparameter Table "public.stoqs_measuredparameter" Column | Type | Modifiers | Storage | Description ----------------+------------------+----------------------------------------------------------------------+---------+------------- id | integer | not null default nextval('stoqs_measuredparameter_id_seq'::regclass) | plain | measurement_id | integer | not null | plain | parameter_id | integer | not null | plain | datavalue | double precision | not null | plain | Indexes: "stoqs_measuredparameter_pkey" PRIMARY KEY, btree (id) "stoqs_measuredparameter_measurement_id_parameter_id_key" UNIQUE CONSTRAINT, btree (measurement_id, parameter_id) "stoqs_measuredparameter_datavalue" btree (datavalue) "stoqs_measuredparameter_measurement_id" btree (measurement_id) "stoqs_measuredparameter_parameter_id" btree (parameter_id) Foreign-key constraints: "stoqs_measuredparameter_measurement_id_fkey" FOREIGN KEY (measurement_id) REFERENCES stoqs_measurement(id) DEFERRABLE INITIALLY DEFERRED "stoqs_measuredparameter_parameter_id_fkey" FOREIGN KEY (parameter_id) REFERENCES stoqs_parameter(id) DEFERRABLE INITIALLY DEFERRED Has OIDs: no Thanks for the suggestion and advice to examine the web app performance. We've actually taken quite a few steps to optimize how the web app works. The example query I provided is a simple worst-case one that we can use to help us decide on the proper hardware. An actual query performed by the web app is: stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_march2013_s-# stoqs_parameter.name AS parameter__name, stoqs_march2013_s-# stoqs_parameter.standard_name AS parameter__standard_name, stoqs_march2013_s-# stoqs_measurement.depth AS measurement__depth, stoqs_march2013_s-# stoqs_measurement.geom AS measurement__geom, stoqs_march2013_s-# stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_march2013_s-# stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_march2013_s-# stoqs_measuredparameter.datavalue AS datavalue, stoqs_march2013_s-# stoqs_parameter.units AS parameter__units stoqs_march2013_s-# FROM stoqs_parameter p1, stoqs_march2013_s-# stoqs_measuredparameter stoqs_march2013_s-# INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) stoqs_march2013_s-# INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) stoqs_march2013_s-# INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) stoqs_march2013_s-# INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) stoqs_march2013_s-# INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) stoqs_march2013_s-# INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id stoqs_march2013_s-# WHERE (p1.name = 'sea_water_sigma_t') stoqs_march2013_s-# AND (mp1.datavalue > 25.19) stoqs_march2013_s-# AND (mp1.datavalue < 26.01) stoqs_march2013_s-# AND (mp1.parameter_id = p1.id) stoqs_march2013_s-# AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' stoqs_march2013_s(# AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' stoqs_march2013_s(# AND stoqs_parameter.name IN ('fl700_uncorr') stoqs_march2013_s(# AND stoqs_measurement.depth >= -1.88 stoqs_march2013_s(# AND stoqs_platform.name IN ('dorado') stoqs_march2013_s(# AND stoqs_measurement.depth <= 83.57) stoqs_march2013_s-# ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=770.211..770.211 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=770.200..770.200 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=770.036..770.036 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=91.369..680.072 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=91.346..577.633 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=91.280..531.408 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=91.202..116.140 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.002..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=91.194..109.846 rows=20170 loop s=1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=90.794..90.794 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=62.769..62.769 rows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=27.412..2 7.412 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.008..0. 017 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.012..0.012 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.007..0.010 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.004..0.004 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.004..0.004 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.151..0.151 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.035..0.145 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.005..0.069 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.014..0.014 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.008..0.012 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 770.445 ms (42 rows) We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Thanks, Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} Another response: {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 6:09:58 PM PDT To: Mike McCann Cc: Jeff Janes , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden On Mon, May 13, 2013 at 5:58 PM, Mike McCann wrote: We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Bad assumption. If your real workload will be queries like the one here that takes 700 ms, but you'll be running 10,000 of them a second, you're tuning / hardware choices are going to be much different then if your query is going to be the previous 7 second one. Use realistic queries, not ones that are nothing like what your real ones will be. then use pgbench and its ability to run custom sql scripts to get a REAL idea how your hardware performs. Note that if you will run the slow query you posted like once a minute and roll it up or cache it then don't get too worried about it. Pay attention to the queries that will add up, in aggregate, to your greatest load. {noformat} ]]> 21364743 9831200 9798449 9142275 2009-02-04 07:57:57.457 2009-02-04 07:57:57.457 9831195 9798444 18284595 18251828 10388133 10355402 1867879 1835115 1867880 1835116 1867881 1835117 1867882 1835118 21365522 21398244 2 2013-07-16 12:05:55.177 2013-07-16 12:09:34.157 21365488 9831183 9798432 1212638 1179873 10388159 10355428 1212658 1179893 8946080 8913331 21398684 cd /mbari/Tempbox/mccann <103 elvis.shore.mbari.org /Tempbox/mccann> tar cvf 2012.268.07.tar 2012.268.07 2012.268.07/ 2012.268.07/SmartSamplerDataLog.log 2012.268.07/Gulper.log ... 2012.268.07/ctdDriver2.cfg 2012.268.07/gps.log 2012.268.07/m3dmgx1.log <105 elvis.shore.mbari.org /Tempbox/mccann> gzip 2012.268.07.tar {noformat} # Copy tarball to /usr/local/auv_ctd and run the auvportal which submits the data to SSDS: {noformat} <255 elvis.shore.mbari.org /Tempbox/mccann> cp 2012.268.07.tar.gz /usr/local/auv_ctd <257 elvis.shore.mbari.org /local/auv_ctd> ./portal AUVCTD2012268 2012.268.07.tar.gz Tue, 25 Sep 2012 09:27:43 PDT: processConnection() got message: AUVCTD2012268;2012.268.07.tar.gz dstDir.getName() = 2012268 Unpacking /usr/local/auv_ctd/2012.268.07.tar.gz Copying /usr/local/auv_ctd/2012268/2012.268.07/ctdDriver2.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/ctdDriver2.log Copying /usr/local/auv_ctd/2012268/2012.268.07/gps.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/gps.log Copying /usr/local/auv_ctd/2012268/2012.268.07/isuslog.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/isuslog.log Copying /usr/local/auv_ctd/2012268/2012.268.07/devices.cfg to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/devices.cfg ... DEBUG [Thread-0] (XmlBuilder.java:538) - appendPrimitiveType called with method name isValidRole on object of class moos.ssds.metadata.DataProducer DEBUG [Thread-0] (XmlBuilder.java:482) - appendElement called on class moos.ssds.metadata.DataProducer using method getClass DEBUG [Thread-0] (XmlBuilder.java:278) - Converting java.lang.Class(class moos.ssds.metadata.DataProducer) to XML Marshalled the AUV mission in /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 into XML Attempting to publish AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ published XML describing mission AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ If everything goes well in SSDS:Ingest/Ruminate an email message to auvctd@listserver.mbari.org will arrive shortly. Tue, 25 Sep 2012 09:34:29 PDT: Done receiving AUV mission data. {noformat} Note: With the move of the portal to from jeckel to elvis the mailing to auvctd@listserver.mbari.org has stopped working. This needs to be debugged and fixed. # Process the LOPC data, e.g.: {noformat} <109 elvis.shore.mbari.org /u/ssdsadmin> cd ~/dev/auv_ctd/src/py/auv/ <110 elvis.shore.mbari.org /py/auv> processLOPC.pl 2013.224.02 {noformat} # Once the mission's instrument netcdf files have been created (check pismo:/opt/ssds/logs/ssds-updateBot.log for successful processing) the overall Dorado science data processing script can be edited with the new survey to be processed: {noformat} <102 elvis.shore.mbari.org /u/ssdsadmin> cd dev/auv_ctd/src/matlab/doradosdp/process/ <103 elvis.shore.mbari.org /doradosdp/process> vi reprocess_surveys.m {noformat} # Add a line for the new survey to the end of the reprocess_surveys.m file, commenting out the previous survey that was processed: {noformat} % CANON September 2012 - laps around ESP drifter - last one for this campaign %%AUVprocess_main('AUVCTD', 2012, ['258.00'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') % CANON September 2012 - laps around ESP drifter - last one for this campaign AUVprocess_main('AUVCTD', 2012, ['268.07'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') {noformat} # Execute the Matlab job with a shell script that properly sets the DISPLAY environment and executes reprocess_surveys.m with matlab: {noformat} <104 elvis.shore.mbari.org /doradosdp/process> RunReprocessJob {noformat} # Check for error messages. If everything works an email is sent to the notifyEmail setting in reprocess_surveys.m, usually auvctd@listserver.mbari.org. h2. Software checkout and repository code management The Software for both the AUV Portal and the Science data processing are located in the _auv_ctd_ module on the CVS server moonjelly.shore.mbari.org. To check the software to a system use your favorite CVS client, or on Unix use these commands: {noformat} cd cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). {noformat} Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): {noformat} {noformat} > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... {noformat} h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: {noformat} usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy {noformat} For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: {noformat} adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 {noformat} Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look something like this: {noformat} bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ {noformat} \\ On the portal computer the screen will display text something like this: {noformat} Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS {noformat} Note: Detailed command-line instructions for "portaling" AUVCTD mission data into SSDS are available in the README file in the project or in the operational location on jeckel.shore.mbari.org in /usr/local/auv_ctd/ directory. There is also the portal.log file there which logs the output of the deamon process running on jeckel that receives and processes the data from AUV uploads from the vehicle. The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: {noformat} > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> {noformat} So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: {noformat} Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds {noformat} +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables and an Ocean Data View spreadsheet file. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file). If one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files.]]> 21365979 8946078 8913329 8946077 8913328 1867851 1835087 1867852 1835088 10388099 10355368 1867850 1835086 10388101 10355370 10388102 10355371 9405003 9372267 21365487 21398209 1 2013-07-12 22:16:53.933 2013-07-12 22:16:53.933 21365482 21398682 cd /mbari/Tempbox/mccann <103 elvis.shore.mbari.org /Tempbox/mccann> tar cvf 2012.268.07.tar 2012.268.07 2012.268.07/ 2012.268.07/SmartSamplerDataLog.log 2012.268.07/Gulper.log ... 2012.268.07/ctdDriver2.cfg 2012.268.07/gps.log 2012.268.07/m3dmgx1.log <105 elvis.shore.mbari.org /Tempbox/mccann> gzip 2012.268.07.tar {noformat} # Copy tarball to /usr/local/auv_ctd and run the auvportal which submits the data to SSDS: {noformat} <255 elvis.shore.mbari.org /Tempbox/mccann> cp 2012.268.07.tar.gz /usr/local/auv_ctd <257 elvis.shore.mbari.org /local/auv_ctd> ./portal AUVCTD2012268 2012.268.07.tar.gz Tue, 25 Sep 2012 09:27:43 PDT: processConnection() got message: AUVCTD2012268;2012.268.07.tar.gz dstDir.getName() = 2012268 Unpacking /usr/local/auv_ctd/2012.268.07.tar.gz Copying /usr/local/auv_ctd/2012268/2012.268.07/ctdDriver2.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/ctdDriver2.log Copying /usr/local/auv_ctd/2012268/2012.268.07/gps.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/gps.log Copying /usr/local/auv_ctd/2012268/2012.268.07/isuslog.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/isuslog.log Copying /usr/local/auv_ctd/2012268/2012.268.07/devices.cfg to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/devices.cfg ... DEBUG [Thread-0] (XmlBuilder.java:538) - appendPrimitiveType called with method name isValidRole on object of class moos.ssds.metadata.DataProducer DEBUG [Thread-0] (XmlBuilder.java:482) - appendElement called on class moos.ssds.metadata.DataProducer using method getClass DEBUG [Thread-0] (XmlBuilder.java:278) - Converting java.lang.Class(class moos.ssds.metadata.DataProducer) to XML Marshalled the AUV mission in /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 into XML Attempting to publish AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ published XML describing mission AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ If everything goes well in SSDS:Ingest/Ruminate an email message to auvctd@listserver.mbari.org will arrive shortly. Tue, 25 Sep 2012 09:34:29 PDT: Done receiving AUV mission data. {noformat} Note: With the move of the portal to from jeckel to elvis the mailing to auvctd@listserver.mbari.org has stopped working. This needs to be debugged and fixed. # Once the mission's instrument netcdf files have been created (check pismo:/opt/ssds/logs/ssds-updateBot.log for successful processing) the overall Dorado science data processing script can be edited with the new survey to be processed: {noformat} <102 elvis.shore.mbari.org /u/ssdsadmin> cd dev/auv_ctd/src/matlab/doradosdp/process/ <103 elvis.shore.mbari.org /doradosdp/process> vi reprocess_surveys.m {noformat} # Add a line for the new survey to the end of the reprocess_surveys.m file, commenting out the previous survey that was processed: {noformat} % CANON September 2012 - laps around ESP drifter - last one for this campaign %%AUVprocess_main('AUVCTD', 2012, ['258.00'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') % CANON September 2012 - laps around ESP drifter - last one for this campaign AUVprocess_main('AUVCTD', 2012, ['268.07'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') {noformat} # Execute the Matlab job with a shell script that properly sets the DISPLAY environment and executes reprocess_surveys.m with matlab: {noformat} <104 elvis.shore.mbari.org /doradosdp/process> RunReprocessJob {noformat} # Check for error messages. If everything works an email is sent to the notifyEmail setting in reprocess_surveys.m, usually auvctd@listserver.mbari.org. h2. Software checkout and repository code management The Software for both the AUV Portal and the Science data processing are located in the _auv_ctd_ module on the CVS server moonjelly.shore.mbari.org. To check the software to a system use your favorite CVS client, or on Unix use these commands: {noformat} cd cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). {noformat} Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): {noformat} {noformat} > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... {noformat} h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: {noformat} usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy {noformat} For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: {noformat} adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 {noformat} Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look something like this: {noformat} bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ {noformat} \\ On the portal computer the screen will display text something like this: {noformat} Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS {noformat} Note: Detailed command-line instructions for "portaling" AUVCTD mission data into SSDS are available in the README file in the project or in the operational location on jeckel.shore.mbari.org in /usr/local/auv_ctd/ directory. There is also the portal.log file there which logs the output of the deamon process running on jeckel that receives and processes the data from AUV uploads from the vehicle. The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: {noformat} > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> {noformat} So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: {noformat} Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds {noformat} +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables and an Ocean Data View spreadsheet file. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file). If one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files.]]> 21365977 21398679 cd /mbari/Tempbox/mccann <103 elvis.shore.mbari.org /Tempbox/mccann> tar cvf 2012.268.07.tar 2012.268.07 2012.268.07/ 2012.268.07/SmartSamplerDataLog.log 2012.268.07/Gulper.log ... 2012.268.07/ctdDriver2.cfg 2012.268.07/gps.log 2012.268.07/m3dmgx1.log <105 elvis.shore.mbari.org /Tempbox/mccann> gzip 2012.268.07.tar {noformat} # Copy tarball to /usr/local/auv_ctd and run the auvportal which submits the data to SSDS: {noformat} <255 elvis.shore.mbari.org /Tempbox/mccann> cp 2012.268.07.tar.gz /usr/local/auv_ctd <257 elvis.shore.mbari.org /local/auv_ctd> ./portal AUVCTD2012268 2012.268.07.tar.gz Tue, 25 Sep 2012 09:27:43 PDT: processConnection() got message: AUVCTD2012268;2012.268.07.tar.gz dstDir.getName() = 2012268 Unpacking /usr/local/auv_ctd/2012.268.07.tar.gz Copying /usr/local/auv_ctd/2012268/2012.268.07/ctdDriver2.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/ctdDriver2.log Copying /usr/local/auv_ctd/2012268/2012.268.07/gps.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/gps.log Copying /usr/local/auv_ctd/2012268/2012.268.07/isuslog.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/isuslog.log Copying /usr/local/auv_ctd/2012268/2012.268.07/devices.cfg to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/devices.cfg ... DEBUG [Thread-0] (XmlBuilder.java:538) - appendPrimitiveType called with method name isValidRole on object of class moos.ssds.metadata.DataProducer DEBUG [Thread-0] (XmlBuilder.java:482) - appendElement called on class moos.ssds.metadata.DataProducer using method getClass DEBUG [Thread-0] (XmlBuilder.java:278) - Converting java.lang.Class(class moos.ssds.metadata.DataProducer) to XML Marshalled the AUV mission in /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 into XML Attempting to publish AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ published XML describing mission AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ If everything goes well in SSDS:Ingest/Ruminate an email message to auvctd@listserver.mbari.org will arrive shortly. Tue, 25 Sep 2012 09:34:29 PDT: Done receiving AUV mission data. {noformat} Note: With the move of the portal to from jeckel to elvis the mailing to auvctd@listserver.mbari.org has stopped working. This needs to be debugged and fixed. # Once the mission's instrument netcdf files have been created (check pismo:/opt/ssds/logs/ssds-updateBot.log for successful processing) the overall Dorado science data processing script can be edited with the new survey to be processed: {noformat} <102 elvis.shore.mbari.org /u/ssdsadmin> cd dev/auv_ctd/src/matlab/doradosdp/process/ <103 elvis.shore.mbari.org /doradosdp/process> vi reprocess_surveys.m {noformat} # Add a line for the new survey to the end of the reprocess_surveys.m file, commenting out the previous survey that was processed: {noformat} % CANON September 2012 - laps around ESP drifter - last one for this campaign %%AUVprocess_main('AUVCTD', 2012, ['258.00'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') % CANON September 2012 - laps around ESP drifter - last one for this campaign AUVprocess_main('AUVCTD', 2012, ['268.07'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') {noformat} # Execute the Matlab job with a shell script that properly sets the DISPLAY environment and executes reprocess_surveys.m with matlab: {noformat} <104 elvis.shore.mbari.org /doradosdp/process> RunReprocessJob {noformat} # Check for error messages. If everything works an email is sent to the notifyEmail setting in reprocess_surveys.m, usually auvctd@listserver.mbari.org. h2. Software checkout and repository code management The Software for both the AUV Portal and the Science data processing are located in the _auv_ctd_ module on the CVS server moonjelly.shore.mbari.org. To check the software to a system use your favorite CVS client, or on Unix use these commands: {noformat} cd cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). {noformat} Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): {noformat} {noformat} > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... {noformat} h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: {noformat} usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy {noformat} For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: {noformat} adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 {noformat} Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look something like this: {noformat} bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ {noformat} \\ On the portal computer the screen will display text something like this: {noformat} Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS {noformat} Note: Detailed command-line instructions for "portaling" AUVCTD mission data into SSDS are available in the README file in the project or in the operational location on jeckel.shore.mbari.org in /usr/local/auv_ctd/ directory. There is also the portal.log file there which logs the output of the deamon process running on jeckel that receives and processes the data from AUV uploads from the vehicle. The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: {noformat} > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> {noformat} So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: {noformat} Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds {noformat} +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables and an Ocean Data View spreadsheet file. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file). If one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files.]]> 21365974 21365489 21398211 1 2013-07-16 12:05:55.177 2013-07-16 12:05:55.177 21365488 21398680 cd /mbari/Tempbox/mccann <103 elvis.shore.mbari.org /Tempbox/mccann> tar cvf 2012.268.07.tar 2012.268.07 2012.268.07/ 2012.268.07/SmartSamplerDataLog.log 2012.268.07/Gulper.log ... 2012.268.07/ctdDriver2.cfg 2012.268.07/gps.log 2012.268.07/m3dmgx1.log <105 elvis.shore.mbari.org /Tempbox/mccann> gzip 2012.268.07.tar {noformat} # Copy tarball to /usr/local/auv_ctd and run the auvportal which submits the data to SSDS: {noformat} <255 elvis.shore.mbari.org /Tempbox/mccann> cp 2012.268.07.tar.gz /usr/local/auv_ctd <257 elvis.shore.mbari.org /local/auv_ctd> ./portal AUVCTD2012268 2012.268.07.tar.gz Tue, 25 Sep 2012 09:27:43 PDT: processConnection() got message: AUVCTD2012268;2012.268.07.tar.gz dstDir.getName() = 2012268 Unpacking /usr/local/auv_ctd/2012.268.07.tar.gz Copying /usr/local/auv_ctd/2012268/2012.268.07/ctdDriver2.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/ctdDriver2.log Copying /usr/local/auv_ctd/2012268/2012.268.07/gps.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/gps.log Copying /usr/local/auv_ctd/2012268/2012.268.07/isuslog.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/isuslog.log Copying /usr/local/auv_ctd/2012268/2012.268.07/devices.cfg to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/devices.cfg ... DEBUG [Thread-0] (XmlBuilder.java:538) - appendPrimitiveType called with method name isValidRole on object of class moos.ssds.metadata.DataProducer DEBUG [Thread-0] (XmlBuilder.java:482) - appendElement called on class moos.ssds.metadata.DataProducer using method getClass DEBUG [Thread-0] (XmlBuilder.java:278) - Converting java.lang.Class(class moos.ssds.metadata.DataProducer) to XML Marshalled the AUV mission in /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 into XML Attempting to publish AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ published XML describing mission AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ If everything goes well in SSDS:Ingest/Ruminate an email message to auvctd@listserver.mbari.org will arrive shortly. Tue, 25 Sep 2012 09:34:29 PDT: Done receiving AUV mission data. {noformat} Note: With the move of the portal to from jeckel to elvis the mailing to auvctd@listserver.mbari.org has stopped working. This needs to be debugged and fixed. # Once the mission's instrument netcdf files have been created (check pismo:/opt/ssds/logs/ssds-updateBot.log for successful processing) the overall Dorado science data processing script can be edited with the new survey to be processed: {noformat} <102 elvis.shore.mbari.org /u/ssdsadmin> cd dev/auv_ctd/src/matlab/doradosdp/process/ <103 elvis.shore.mbari.org /doradosdp/process> vi reprocess_surveys.m {noformat} # Add a line for the new survey to the end of the reprocess_surveys.m file, commenting out the previous survey that was processed: {noformat} % CANON September 2012 - laps around ESP drifter - last one for this campaign %%AUVprocess_main('AUVCTD', 2012, ['258.00'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') % CANON September 2012 - laps around ESP drifter - last one for this campaign AUVprocess_main('AUVCTD', 2012, ['268.07'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') {noformat} # Execute the Matlab job with a shell script that properly sets the DISPLAY environment and executes reprocess_surveys.m with matlab: {noformat} <104 elvis.shore.mbari.org /doradosdp/process> RunReprocessJob {noformat} # Check for error messages. If everything works an email is sent to the notifyEmail setting in reprocess_surveys.m, usually auvctd@listserver.mbari.org. h2. Software checkout and repository code management The Software for both the AUV Portal and the Science data processing are located in the _auv_ctd_ module on the CVS server moonjelly.shore.mbari.org. To check the software to a system use your favorite CVS client, or on Unix use these commands: {noformat} cd cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). {noformat} Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): {noformat} {noformat} > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... {noformat} h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: {noformat} usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy {noformat} For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: {noformat} adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 {noformat} Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look something like this: {noformat} bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ {noformat} \\ On the portal computer the screen will display text something like this: {noformat} Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS {noformat} Note: Detailed command-line instructions for "portaling" AUVCTD mission data into SSDS are available in the README file in the project or in the operational location on jeckel.shore.mbari.org in /usr/local/auv_ctd/ directory. There is also the portal.log file there which logs the output of the deamon process running on jeckel that receives and processes the data from AUV uploads from the vehicle. The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: {noformat} > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> {noformat} So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: {noformat} Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds {noformat} +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables and an Ocean Data View spreadsheet file. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file). If one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files.]]> 21365975 16810178 16777416 21398669 cd /mbari/Tempbox/mccann <103 elvis.shore.mbari.org /Tempbox/mccann> tar cvf 2012.268.07.tar 2012.268.07 2012.268.07/ 2012.268.07/SmartSamplerDataLog.log 2012.268.07/Gulper.log ... 2012.268.07/ctdDriver2.cfg 2012.268.07/gps.log 2012.268.07/m3dmgx1.log <105 elvis.shore.mbari.org /Tempbox/mccann> gzip 2012.268.07.tar {noformat} # Copy tarball to /usr/local/auv_ctd and run the auvportal which submits the data to SSDS: {noformat} <255 elvis.shore.mbari.org /Tempbox/mccann> cp 2012.268.07.tar.gz /usr/local/auv_ctd <257 elvis.shore.mbari.org /local/auv_ctd> ./portal AUVCTD2012268 2012.268.07.tar.gz Tue, 25 Sep 2012 09:27:43 PDT: processConnection() got message: AUVCTD2012268;2012.268.07.tar.gz dstDir.getName() = 2012268 Unpacking /usr/local/auv_ctd/2012.268.07.tar.gz Copying /usr/local/auv_ctd/2012268/2012.268.07/ctdDriver2.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/ctdDriver2.log Copying /usr/local/auv_ctd/2012268/2012.268.07/gps.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/gps.log Copying /usr/local/auv_ctd/2012268/2012.268.07/isuslog.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/isuslog.log Copying /usr/local/auv_ctd/2012268/2012.268.07/devices.cfg to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/devices.cfg ... DEBUG [Thread-0] (XmlBuilder.java:538) - appendPrimitiveType called with method name isValidRole on object of class moos.ssds.metadata.DataProducer DEBUG [Thread-0] (XmlBuilder.java:482) - appendElement called on class moos.ssds.metadata.DataProducer using method getClass DEBUG [Thread-0] (XmlBuilder.java:278) - Converting java.lang.Class(class moos.ssds.metadata.DataProducer) to XML Marshalled the AUV mission in /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 into XML Attempting to publish AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ published XML describing mission AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ If everything goes well in SSDS:Ingest/Ruminate an email message to auvctd@listserver.mbari.org will arrive shortly. Tue, 25 Sep 2012 09:34:29 PDT: Done receiving AUV mission data. {noformat} Note: With the move of the portal to from jeckel to elvis the mailing to auvctd@listserver.mbari.org has stopped working. This needs to be debugged and fixed. # Once the mission's instrument netcdf files have been created (check pismo:/opt/ssds/logs/ssds-updateBot.log for successful processing) the overall Dorado science data processing script can be edited with the new survey to be processed: {noformat} <102 elvis.shore.mbari.org /u/ssdsadmin> cd dev/auv_ctd/src/matlab/doradosdp/process/ <103 elvis.shore.mbari.org /doradosdp/process> vi reprocess_surveys.m {noformat} # Add a line for the new survey to the end of the reprocess_surveys.m file, commenting out the previous survey that was processed: {noformat} % CANON September 2012 - laps around ESP drifter - last one for this campaign %%AUVprocess_main('AUVCTD', 2012, ['258.00'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') % CANON September 2012 - laps around ESP drifter - last one for this campaign AUVprocess_main('AUVCTD', 2012, ['268.07'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') {noformat} # Execute the Matlab job with a shell script that properly sets the DISPLAY environment and executes reprocess_surveys.m with matlab: {noformat} <104 elvis.shore.mbari.org /doradosdp/process> RunReprocessJob {noformat} # Check for error messages. If everything works an email is sent to the notifyEmail setting in reprocess_surveys.m, usually auvctd@listserver.mbari.org. h2. Software checkout and repository code management The Software for both the AUV Portal and the Science data processing are located in the _auv_ctd_ module on the CVS server moonjelly.shore.mbari.org. To check the software to a system use your favorite CVS client, or on Unix use these commands: {noformat} cd cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). {noformat} Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): {noformat} {noformat} > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... {noformat} h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: {noformat} usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy {noformat} For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: {noformat} adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 {noformat} Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look something like this: {noformat} bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ {noformat} \\ On the portal computer the screen will display text something like this: {noformat} Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS {noformat} Note: Detailed command-line instructions for "portaling" AUVCTD mission data into SSDS are available in the README file in the project or in the operational location on jeckel.shore.mbari.org in /usr/local/auv_ctd/ directory. There is also the portal.log file there which logs the output of the deamon process running on jeckel that receives and processes the data from AUV uploads from the vehicle. The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: {noformat} > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> {noformat} So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: {noformat} Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds {noformat} +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables and an Ocean Data View spreadsheet file. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file). If one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files.]]> 21365964 21398668 cd /mbari/Tempbox/mccann <103 elvis.shore.mbari.org /Tempbox/mccann> tar cvf 2012.268.07.tar 2012.268.07 2012.268.07/ 2012.268.07/SmartSamplerDataLog.log 2012.268.07/Gulper.log ... 2012.268.07/ctdDriver2.cfg 2012.268.07/gps.log 2012.268.07/m3dmgx1.log <105 elvis.shore.mbari.org /Tempbox/mccann> gzip 2012.268.07.tar {noformat} # Copy tarball to /usr/local/auv_ctd and run the auvportal which submits the data to SSDS: {noformat} <255 elvis.shore.mbari.org /Tempbox/mccann> cp 2012.268.07.tar.gz /usr/local/auv_ctd <257 elvis.shore.mbari.org /local/auv_ctd> ./portal AUVCTD2012268 2012.268.07.tar.gz Tue, 25 Sep 2012 09:27:43 PDT: processConnection() got message: AUVCTD2012268;2012.268.07.tar.gz dstDir.getName() = 2012268 Unpacking /usr/local/auv_ctd/2012.268.07.tar.gz Copying /usr/local/auv_ctd/2012268/2012.268.07/ctdDriver2.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/ctdDriver2.log Copying /usr/local/auv_ctd/2012268/2012.268.07/gps.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/gps.log Copying /usr/local/auv_ctd/2012268/2012.268.07/isuslog.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/isuslog.log Copying /usr/local/auv_ctd/2012268/2012.268.07/devices.cfg to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/devices.cfg ... DEBUG [Thread-0] (XmlBuilder.java:538) - appendPrimitiveType called with method name isValidRole on object of class moos.ssds.metadata.DataProducer DEBUG [Thread-0] (XmlBuilder.java:482) - appendElement called on class moos.ssds.metadata.DataProducer using method getClass DEBUG [Thread-0] (XmlBuilder.java:278) - Converting java.lang.Class(class moos.ssds.metadata.DataProducer) to XML Marshalled the AUV mission in /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 into XML Attempting to publish AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ published XML describing mission AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ If everything goes well in SSDS:Ingest/Ruminate an email message to auvctd@listserver.mbari.org will arrive shortly. Tue, 25 Sep 2012 09:34:29 PDT: Done receiving AUV mission data. {noformat} Note: With the move of the portal to from jeckel to elvis the mailing to auvctd@listserver.mbari.org has stopped working. This needs to be debugged and fixed. # Once the mission's instrument netcdf files have been created (check pismo:/opt/ssds/logs/ssds-updateBot.log for successful processing) the overall Dorado science data processing script can be edited with the new survey to be processed: {noformat} <102 elvis.shore.mbari.org /u/ssdsadmin> cd dev/auv_ctd/src/matlab/doradosdp/process/ <103 elvis.shore.mbari.org /doradosdp/process> vi reprocess_surveys.m {noformat} # Add a line for the new survey to the end of the reprocess_surveys.m file, commenting out the previous survey that was processed: {noformat} % CANON September 2012 - laps around ESP drifter - last one for this campaign %%AUVprocess_main('AUVCTD', 2012, ['258.00'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') % CANON September 2012 - laps around ESP drifter - last one for this campaign AUVprocess_main('AUVCTD', 2012, ['268.07'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') {noformat} # Execute the Matlab job with a shell script that properly sets the DISPLAY environment and executes reprocess_surveys.m with matlab: {noformat} <104 elvis.shore.mbari.org /doradosdp/process> RunReprocessJob {noformat} # Check for error messages. If everything works an email is sent to the notifyEmail setting in reprocess_surveys.m, usually auvctd@listserver.mbari.org. h2. Software checkout and repository code management The Software for both the AUV Portal and the Science data processing are located in the _auv_ctd_ module on the CVS server moonjelly.shore.mbari.org. To check the software to a system use your favorite CVS client, or on Unix use these commands: {noformat} cd cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). {noformat} Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): {noformat} {noformat} > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... {noformat} h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: {noformat} usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy {noformat} For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: {noformat} adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 {noformat} Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look something like this: {noformat} bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ {noformat} \\ On the portal computer the screen will display text something like this: {noformat} Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS {noformat} Note: Detailed command-line instructions for "portaling" AUVCTD mission data into SSDS are available in the README file in the project or in the operational location on jeckel.shore.mbari.org in /usr/local/auv_ctd/ directory. There is also the portal.log file there which logs the output of the deamon process running on jeckel that receives and processes the data from AUV uploads from the vehicle. The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: {noformat} > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> {noformat} So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: {noformat} Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds {noformat} +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables and an Ocean Data View spreadsheet file. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file). If one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files.]]> 21365963 2195833 2163069 2195834 2163070 2195832 2163068 21398662 cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). {noformat} Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): {noformat} {noformat} > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... {noformat} h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: {noformat} usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy {noformat} For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: {noformat} adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 {noformat} Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look something like this: {noformat} bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ {noformat} \\ On the portal computer the screen will display text something like this: {noformat} Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS {noformat} Note: Detailed command-line instructions for "portaling" AUVCTD mission data into SSDS are available in the README file in the project or in the operational location on jeckel.shore.mbari.org in /usr/local/auv_ctd/ directory. There is also the portal.log file there which logs the output of the deamon process running on jeckel that receives and processes the data from AUV uploads from the vehicle. The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: {noformat} > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> {noformat} So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: {noformat} Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds {noformat} +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables and an Ocean Data View spreadsheet file. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file). If one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files. ]]> 21365957 21398666 cd /mbari/Tempbox/mccann <103 elvis.shore.mbari.org /Tempbox/mccann> tar cvf 2012.268.07.tar 2012.268.07 2012.268.07/ 2012.268.07/SmartSamplerDataLog.log 2012.268.07/Gulper.log ... 2012.268.07/ctdDriver2.cfg 2012.268.07/gps.log 2012.268.07/m3dmgx1.log <105 elvis.shore.mbari.org /Tempbox/mccann> gzip 2012.268.07.tar {noformat} # Copy tarball to /usr/local/auv_ctd and run the auvportal which submits the data to SSDS: {noformat} <255 elvis.shore.mbari.org /Tempbox/mccann> cp 2012.268.07.tar.gz /usr/local/auv_ctd <257 elvis.shore.mbari.org /local/auv_ctd> ./portal AUVCTD2012268 2012.268.07.tar.gz CP = ./:./activation.jar:./auvctd_doc.jar:./auvctd.jar:./auvctd_src.jar:./commons-lang-2.0.jar:./commons-logging.jar:./db-ojb-1.0.1.jar:./ftp.jar:./jbossall-client.jar:./junit.jar:./ledatastream.jar:./log4j.jar:./netcdf.jar:./opendap.jar:./siam.jar:./ssds-client-pub-new-ssds.jar:./ssds.jar:./ssds-services-metadata-client-new-ssds.jar:./tar.jar:./xercesImpl.jar:./xmlParserAPIs.jar:./xom-1.0b5.jar Tue, 25 Sep 2012 09:27:43 PDT: processConnection() got message: AUVCTD2012268;2012.268.07.tar.gz dstDir.getName() = 2012268 Unpacking /usr/local/auv_ctd/2012.268.07.tar.gz Copying /usr/local/auv_ctd/2012268/2012.268.07/ctdDriver2.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/ctdDriver2.log Copying /usr/local/auv_ctd/2012268/2012.268.07/gps.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/gps.log Copying /usr/local/auv_ctd/2012268/2012.268.07/isuslog.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/isuslog.log Copying /usr/local/auv_ctd/2012268/2012.268.07/devices.cfg to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/devices.cfg ... DEBUG [Thread-0] (XmlBuilder.java:538) - appendPrimitiveType called with method name isValidRole on object of class moos.ssds.metadata.DataProducer DEBUG [Thread-0] (XmlBuilder.java:482) - appendElement called on class moos.ssds.metadata.DataProducer using method getClass DEBUG [Thread-0] (XmlBuilder.java:278) - Converting java.lang.Class(class moos.ssds.metadata.DataProducer) to XML Marshalled the AUV mission in /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 into XML Attempting to publish AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ published XML describing mission AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ If everything goes well in SSDS:Ingest/Ruminate an email message to auvctd@listserver.mbari.org will arrive shortly. Tue, 25 Sep 2012 09:34:29 PDT: Done receiving AUV mission data. {noformat} Note: With the move of the portal to from jeckel to elvis the mailing to auvctd@listserver.mbari.org has stopped working. This needs to be debugged and fixed. # Once the mission's instrument netcdf files have been created (check pismo:/opt/ssds/logs/ssds-updateBot.log for successful processing) the overall Dorado science data processing script can be edited with the new survey to be processed: {noformat} <102 elvis.shore.mbari.org /u/ssdsadmin> cd dev/auv_ctd/src/matlab/doradosdp/process/ <103 elvis.shore.mbari.org /doradosdp/process> vi reprocess_surveys.m {noformat} # Add a line for the new survey to the end of the reprocess_surveys.m file, commenting out the previous survey that was processed: {noformat} % CANON September 2012 - laps around ESP drifter - last one for this campaign %%AUVprocess_main('AUVCTD', 2012, ['258.00'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') % CANON September 2012 - laps around ESP drifter - last one for this campaign AUVprocess_main('AUVCTD', 2012, ['268.07'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') {noformat} # Execute the Matlab job with a shell script that properly sets the DISPLAY environment and executes reprocess_surveys.m with matlab: {noformat} <104 elvis.shore.mbari.org /doradosdp/process> RunReprocessJob {noformat} # Check for error messages. If everything works an email is sent to the notifyEmail setting in reprocess_surveys.m, usually auvctd@listserver.mbari.org. h2. Software checkout and repository code management The Software for both the AUV Portal and the Science data processing are located in the _auv_ctd_ module on the CVS server moonjelly.shore.mbari.org. To check the software to a system use your favorite CVS client, or on Unix use these commands: {noformat} cd cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). {noformat} Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): {noformat} {noformat} > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... {noformat} h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: {noformat} usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy {noformat} For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: {noformat} adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 {noformat} Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look something like this: {noformat} bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ {noformat} \\ On the portal computer the screen will display text something like this: {noformat} Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS {noformat} Note: Detailed command-line instructions for "portaling" AUVCTD mission data into SSDS are available in the README file in the project or in the operational location on jeckel.shore.mbari.org in /usr/local/auv_ctd/ directory. There is also the portal.log file there which logs the output of the deamon process running on jeckel that receives and processes the data from AUV uploads from the vehicle. The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: {noformat} > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> {noformat} So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: {noformat} Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds {noformat} +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables and an Ocean Data View spreadsheet file. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file). If one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files.]]> 21365961 21398665 cd /mbari/Tempbox/mccann <103 elvis.shore.mbari.org /Tempbox/mccann> tar cvf 2012.268.07.tar 2012.268.07 2012.268.07/ 2012.268.07/SmartSamplerDataLog.log 2012.268.07/Gulper.log ... 2012.268.07/ctdDriver2.cfg 2012.268.07/gps.log 2012.268.07/m3dmgx1.log <105 elvis.shore.mbari.org /Tempbox/mccann> gzip 2012.268.07.tar {noformat} # Copy tarball to /usr/local/auv_ctd and run the auvportal which submits the data to SSDS: {noformat} <255 elvis.shore.mbari.org /Tempbox/mccann> cp 2012.268.07.tar.gz /usr/local/auv_ctd <257 elvis.shore.mbari.org /local/auv_ctd> ./portal AUVCTD2012268 2012.268.07.tar.gz CP = ./:./activation.jar:./auvctd_doc.jar:./auvctd.jar:./auvctd_src.jar:./commons-lang-2.0.jar:./commons-logging.jar:./db-ojb-1.0.1.jar:./ftp.jar:./jbossall-client.jar:./junit.jar:./ledatastream.jar:./log4j.jar:./netcdf.jar:./opendap.jar:./siam.jar:./ssds-client-pub-new-ssds.jar:./ssds.jar:./ssds-services-metadata-client-new-ssds.jar:./tar.jar:./xercesImpl.jar:./xmlParserAPIs.jar:./xom-1.0b5.jar Tue, 25 Sep 2012 09:27:43 PDT: processConnection() got message: AUVCTD2012268;2012.268.07.tar.gz dstDir.getName() = 2012268 Unpacking /usr/local/auv_ctd/2012.268.07.tar.gz Copying /usr/local/auv_ctd/2012268/2012.268.07/ctdDriver2.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/ctdDriver2.log Copying /usr/local/auv_ctd/2012268/2012.268.07/gps.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/gps.log Copying /usr/local/auv_ctd/2012268/2012.268.07/isuslog.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/isuslog.log Copying /usr/local/auv_ctd/2012268/2012.268.07/devices.cfg to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/devices.cfg ... DEBUG [Thread-0] (XmlBuilder.java:538) - appendPrimitiveType called with method name isValidRole on object of class moos.ssds.metadata.DataProducer DEBUG [Thread-0] (XmlBuilder.java:482) - appendElement called on class moos.ssds.metadata.DataProducer using method getClass DEBUG [Thread-0] (XmlBuilder.java:278) - Converting java.lang.Class(class moos.ssds.metadata.DataProducer) to XML Marshalled the AUV mission in /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 into XML Attempting to publish AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ published XML describing mission AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ If everything goes well in SSDS:Ingest/Ruminate an email message to auvctd@listserver.mbari.org will arrive shortly. Tue, 25 Sep 2012 09:34:29 PDT: Done receiving AUV mission data. {noformat} Note: With the move of the portal to from jeckel to elvis the mailing to auvctd@listserver.mbari.org has stopped working. This needs to be debugged and fixed. # Once the mission's instrument netcdf files have been created (check pismo:/opt/ssds/logs/ssds-updateBot.log for successful processing) the overall Dorado science data processing script can be edited with the new survey to be processed: {noformat} <102 elvis.shore.mbari.org /u/ssdsadmin> cd dev/auv_ctd/src/matlab/doradosdp/process/ <103 elvis.shore.mbari.org /doradosdp/process> vi reprocess_surveys.m {noformat} # Add a line for the new survey to the end of the reprocess_surveys.m file, commenting out the previous survey that was processed: {noformat} % CANON September 2012 - laps around ESP drifter - last one for this campaign %%AUVprocess_main('AUVCTD', 2012, ['258.00'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') % CANON September 2012 - laps around ESP drifter - last one for this campaign AUVprocess_main('AUVCTD', 2012, ['268.07'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') {noformat} # Execute the Matlab job with a shell script that properly sets the DISPLAY environment and executes reprocess_surveys.m with matlab: {noformat} <104 elvis.shore.mbari.org /doradosdp/process> RunReprocessJob {noformat} # Check for error messages. If everything works an email is sent to the notifyEmail setting in reprocess_surveys.m, usually auvctd@listserver.mbari.org. h2. Software checkout and repository code management The Software for both the AUV Portal and the Science data processing are located in the _auv_ctd_ module on the CVS server moonjelly.shore.mbari.org. To check the software to a system use your favorite CVS client, or on Unix use these commands: {noformat} cd cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). {noformat} Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): {noformat} {noformat} > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... {noformat} h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: {noformat} usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy {noformat} For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: {noformat} adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 {noformat} Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look something like this: {noformat} bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ {noformat} \\ On the portal computer the screen will display text something like this: {noformat} Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS {noformat} Note: Detailed command-line instructions for "portaling" AUVCTD mission data into SSDS are available in the README file in the project or in the operational location on jeckel.shore.mbari.org in /usr/local/auv_ctd/ directory. There is also the portal.log file there which logs the output of the deamon process running on jeckel that receives and processes the data from AUV uploads from the vehicle. The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: {noformat} > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> {noformat} So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: {noformat} Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds {noformat} +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables and an Ocean Data View spreadsheet file. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file). If one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files.]]> 21365960 21398664 cd /mbari/Tempbox/mccann <103 elvis.shore.mbari.org /Tempbox/mccann> tar cvf 2012.268.07.tar 2012.268.07 2012.268.07/ 2012.268.07/SmartSamplerDataLog.log 2012.268.07/Gulper.log ... 2012.268.07/ctdDriver2.cfg 2012.268.07/gps.log 2012.268.07/m3dmgx1.log <105 elvis.shore.mbari.org /Tempbox/mccann> gzip 2012.268.07.tar {noformat} # Copy tarball to /usr/local/auv_ctd and run the auvportal which submits the data to SSDS: {noformat} <255 elvis.shore.mbari.org /Tempbox/mccann> cp 2012.268.07.tar.gz /usr/local/auv_ctd <257 elvis.shore.mbari.org /local/auv_ctd> ./portal AUVCTD2012268 2012.268.07.tar.gz CP = ./:./activation.jar:./auvctd_doc.jar:./auvctd.jar:./auvctd_src.jar:./commons-lang-2.0.jar:./commons-logging.jar:./db-ojb-1.0.1.jar:./ftp.jar:./jbossall-client.jar:./junit.jar:./ledatastream.jar:./log4j.jar:./netcdf.jar:./opendap.jar:./siam.jar:./ssds-client-pub-new-ssds.jar:./ssds.jar:./ssds-services-metadata-client-new-ssds.jar:./tar.jar:./xercesImpl.jar:./xmlParserAPIs.jar:./xom-1.0b5.jar Tue, 25 Sep 2012 09:27:43 PDT: processConnection() got message: AUVCTD2012268;2012.268.07.tar.gz dstDir.getName() = 2012268 Unpacking /usr/local/auv_ctd/2012.268.07.tar.gz Copying /usr/local/auv_ctd/2012268/2012.268.07/ctdDriver2.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/ctdDriver2.log Copying /usr/local/auv_ctd/2012268/2012.268.07/gps.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/gps.log Copying /usr/local/auv_ctd/2012268/2012.268.07/isuslog.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/isuslog.log Copying /usr/local/auv_ctd/2012268/2012.268.07/devices.cfg to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/devices.cfg ... DEBUG [Thread-0] (XmlBuilder.java:538) - appendPrimitiveType called with method name isValidRole on object of class moos.ssds.metadata.DataProducer DEBUG [Thread-0] (XmlBuilder.java:482) - appendElement called on class moos.ssds.metadata.DataProducer using method getClass DEBUG [Thread-0] (XmlBuilder.java:278) - Converting java.lang.Class(class moos.ssds.metadata.DataProducer) to XML Marshalled the AUV mission in /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 into XML Attempting to publish AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ published XML describing mission AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ If everything goes well in SSDS:Ingest/Ruminate an email message to auvctd@listserver.mbari.org will arrive shortly. Tue, 25 Sep 2012 09:34:29 PDT: Done receiving AUV mission data. {noformat} Note: With the move of the portal to from jeckel to elvis the mailing to auvctd@listserver.mbari.org has stopped working. This needs to be debugged and fixed. # Once the mission's instrument netcdf files have been created (check pismo:/opt/ssds/logs/ssds-updateBot.log for successful processing) the overall Dorado science data processing script can be edited with the new survey to be processed: {noformat} <102 elvis.shore.mbari.org /u/ssdsadmin> cd dev/auv_ctd/src/matlab/doradosdp/process/ <103 elvis.shore.mbari.org /doradosdp/process> vi reprocess_surveys.m {noformat} add a line for the new survey to the end of the reprocess_surveys.m file, commenting out the previous survey that was processed: % CANON September 2012 - laps around ESP drifter - last one for this campaign %%AUVprocess_main('AUVCTD', 2012, ['258.00'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') % CANON September 2012 - laps around ESP drifter - last one for this campaign AUVprocess_main('AUVCTD', 2012, ['268.07'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') Execute the Matlab job with a shell script that properly sets the DISPLAY environment and executes reprocess_surveys.m with matlab: <104 elvis.shore.mbari.org /doradosdp/process> RunReprocessJob Check for error messages. If everything works an email is sent to the notifyEmail setting in reprocess_surveys.m, usually auvctd@listserver.mbari.org. (When I ran the science data processing for 268.07 I did get error messages - I notified Kevin of them with a separate email. After those problems are fixed these steps should be executed again.) h2. Software checkout and repository code management The Software for both the AUV Portal and the Science data processing are located in the _auv_ctd_ module on the CVS server moonjelly.shore.mbari.org. To check the software to a system use your favorite CVS client, or on Unix use these commands: {noformat} cd cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). {noformat} Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): {noformat} {noformat} > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... {noformat} h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: {noformat} usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy {noformat} For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: {noformat} adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 {noformat} Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look something like this: {noformat} bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ {noformat} \\ On the portal computer the screen will display text something like this: {noformat} Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS {noformat} Note: Detailed command-line instructions for "portaling" AUVCTD mission data into SSDS are available in the README file in the project or in the operational location on jeckel.shore.mbari.org in /usr/local/auv_ctd/ directory. There is also the portal.log file there which logs the output of the deamon process running on jeckel that receives and processes the data from AUV uploads from the vehicle. The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: {noformat} > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> {noformat} So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: {noformat} Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds {noformat} +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables and an Ocean Data View spreadsheet file. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file). If one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files.]]> 21365959 16810191 16777429 16810193 16777431 16810194 16777432 9404977 cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look somehting like this: bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ \\ On the portal computer the screen will display text something like this: Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04 WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS Once the AUV files have been submitted to SSDS they are available via SSDS's Hierarchical Ocean Observatory Visualization and Editing System (HOOVES) program. The data from these files may be plotted and retrieved from HOOVES. (Note that the MultiGenerateNetcdfTask that creates the .nc files takes several minutes and that you may need to wait up to an hour before querying SSDS with HOOVES to see the netCDF data files.) | | | *Figure 2*. HOOVES showing AUV Hydroscat instrument data processing after submission of data to SSDS. Clicking on the hydroscatlog.nc button brings up a pane where variables may be selected for plotting and numeric display. | The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey | | | *Figure 3*. HOOVES display of processing tree graph for level 1 and level 2 data products. (Click on image for full processing tree graph.) | To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables (which can be plotted against the original instrument netCDF files as a time series in HOOVES) and an Ocean Data View spreadsheet file. The latter may be loaded into ODV directly from HOOVES. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file), therefore HOOVES supports the registration of the creation of a Collection by an ODV user with permission to write to the \\tornado\AUVCTD\surveys\\odv directory. Clicking on the Level-2 .txt file button will import the survey data into HOOVES. If the user then saves that collection along with section splines back to the surveys\\odv directory then thie "ProcessRun" should be entered into SSDS. This is easily done by the user entering their email address on this form and clicking the Submit ProcessRun button: | | | *Figure 4*. Submitting a ProcessRun from HOOVES that registers the existence of an ODV Collection in SSDS. | If an ODV collection is available then that may be used - saving time in selecting the zoom, coastline layer, and section spline. An example screen shot of launching an ODV Collection from HOOVES is shown below. The operation of ODV is beyond the scope of this document. However, if one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files. | | | *Figure 5*. Launching Ocean Data View from HOOVES for visualization with and interaction with AUV survey data. (Click on image for larger view.) | ]]> 9372239 9404978 9372241 9404975 9372237 1867842 1835078 16810196 16777435 9404944 9372205 9404946 9372207 21365401 21398125 1 2013-07-11 11:17:55.807 2013-07-11 11:17:55.807 21365395 9404954 9372217 9831279 9798533 21365410 21398133 2 2013-07-11 11:17:55.807 2013-07-11 11:40:12.263 21365395 21004754 20971991 21004751 20971988 21004752 20971989 21004749 20971986 21004748 20971985 29720646 29687879 29720645 29687878 29720644 29687877 21004739 20971976 21004770 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h3. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata ]]> 20972007 26673456 26640699 21004769 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata ]]> 20972006 21004768 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed ]]> 20972005 21004766 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) ]]> 20972003 26673453 26640696 21004758 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions]]> 20971995 21004756 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} ]]> 20971993 21004783 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} ]]> 20972020 21004785 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} ]]> 20972022 10388244 10355514 21004781 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata ]]> 20972018 21004771 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h3. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata ]]> 20972008 21004772 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata ]]> 20972009 21004796 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) No joy for immediate increase in performance. After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... ]]> 20972033 21004798 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ ]]> 20972035 21004787 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: yum install postgresql91-contrib.x86_64 ]]> 20972024 21004790 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) No joy for immediate increase in performance. After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Continuing to read Chapter 5... Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... ]]> 20972027 21004789 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: yum install postgresql91-contrib.x86_64 ]]> 20972026 5407181 5374445 1212693 [#frozen Pos] [excel]\n\n" unless -d $ARGV[0]; $showOver = $ARGV[1] || 60; # Show if pos repeats for more times than this $outputExcel = 0; $outputExcel = 1 if ( $ARGV[2] =~ /excel/i ); $| = 1; $deltaT = 5 / 60; # # Open directory and get list of all the raw nav files in it. Handle vnta & tibr # opendir (NAVDIR, $ARGV[0]) || die "Can't open directory $ARGV[0]: $!\n"; if ( $ARGV[0] =~ /vnta/ ) { @navfiles = grep { /vnta.txt$/ } readdir (NAVDIR); } elsif ( $ARGV[0] =~ /tibr/ ) { @navfiles = grep { /tibr.txt$/ } readdir (NAVDIR); } else { die "$0 works only for vnta or tibr directories.\n"; } die "Empty \@navFiles list. Was reading from: $ARGV[0]\n" unless @navfiles; closedir (NAVDIR); # # Print header for output # - the aveSD column is the average standard deviation between the original and edited nav # if the values are 0.00 then the original and edited data are identical. Larger # values will indicate a degraded performance in the nav system. There will natuarlly # be a correlation with performance accuracy and depth. # if ( $outputExcel ) { ##print "File,esecs,Lat,Lon,maxPres,numFreezes,totalDuration,recordDuration,aveSD,editedData\n"; print "File\tesecs\tLat\tLon\tmaxPres\tnumFreezes\ttotalDuration\trecordDuration\taveSD\teditedData\n"; } else { print "File str (hhmmss) end froz Lat froz lon sPres ePres # duration\n"; } # # Loop through all the files and collect stats on frozen positions # foreach ( @navfiles ) { $file = $_; print "file = $file\n" if $debug; if ( $outputExcel ) { } else { print "$file\n"; } calcFreezes(); calcSD(); } # End foreach #------------------------------------ # Subroutines: just to partition the work. No args passed. Use global vars # sub calcFreezes { open (NAVFILE, "$ARGV[0]/${file}") || die "Can't open file $ARGV[0]/${file}: $!\n"; $oldLat = 0; $oldLon = 0; $sameCount = 0; $notSame = 0; $prFile = $file; # Print file string, for listing below $prFile =~ s/vnta.txt//; $timeSum = 0; $recSum = 0; $numFrozen = 0; $maxPres = 0; while ( ) { next unless /^\d/; # Skip rec unless it begins with number # # Read record into my variables # ($Year, $Day, $Time, $Usec, $Lat, $Lon, $East, $North, $Pres, $Head, $Alti, $Pitch, $Roll, $PosFlag, $PresFlag, $HeadFlag, $AltiFlag, $AttitFlag) = split (',', $_); if ( $maxPres < $Pres ) { $maxPres = $Pres; } # # Count records of unchanging Lat & Lon # print "$_\n" if $debug; if ( $Lat == $oldLat && $Lon == $oldLon ) { $sameCount++; if ( $sameCount == 1 ) { $begUsec = $Usec; $begTime = $Time; $begPres = $Pres; } } elsif ( $oldLat != 0 ) { $notSame = 1; $timeDiffSec = $Usec - $begUsec; $duration = sprintf("%6.1f min", $timeDiffSec / 60); } # End if ( $Lat ... # # Print out stats for frozen position after we've counted them # if ( $sameCount > $showOver && $notSame ) { if ( $outputExcel ) { } else { printf ("%10s %d to %d %f %f %5.1f to %5.1F %3d %s\n", $prFile, $begTime, $Time, $oldLat, $oldLon, $begPres, $Pres, $sameCount, $duration); $prFile = ' '; } $recSum += $deltaT * $sameCount; $sameCount = 0; $notSame = 0; $timeSum += $duration; $numFrozen++; ##die "Debugging\n";; } # # Reset counter if not over our $showOver criteria # elsif ( $sameCount <= $showOver && $notSame ) { $sameCount = 0; $notSame = 0; } # End if ( $sameCount > $oldLat = $Lat; $oldLon = $Lon; } # End while (NAVFILE # # Print summary for this day # if ( $outputExcel ) { ##print "$prFile,$Usec,$Lat,$Lon,$maxPres,$numFrozen,$timeSum,", sprintf("%6.f", $recSum); print "$prFile\t$Usec\t$Lat\t$Lon\t$maxPres\t$numFrozen\t$timeSum\t", sprintf("%6.f", $recSum); } else { if ( $numFrozen > 0 ) { printf (" %3d frozen positions over %3d records long lasting total of %5.1f minutes\n", $numFrozen, $showOver, $timeSum); } } # End if ( $outputExcel } # End calcFreezes() sub calcSD { # # Create list of lat & lon of differences for the day # open (NAVFILE, "$ARGV[0]/${file}") || die "Can't open file $ARGV[0]/${file}: $!\n"; $editedFile = $file; $editedFile =~ s/\.txt/edited.txt/; if ( -f "$ARGV[0]/${editedFile}" ) { open (EDITEDNAV, "$ARGV[0]/${editedFile}") || die "Can't open file $ARGV[0]/${editedFile}: $!\n"; } else { print STDERR "Can't open file $ARGV[0]/${editedFile}: $!\n"; print STDERR "Skipping this file.\n"; if ( $outputExcel ) { ##print ",0,NoEditedNav\n"; print "\t0\tNoEditedNav\n"; } return; } undef @lonDiff; undef @latDiff; while ( ($raw = ) && ($edited = ) ) { next unless $raw =~ /^\d/; # Skip rec unless it begins with number # # Read raw record into my variables # my ($Year, $Day, $Time, $Usec, $Lat, $Lon, $East, $North, $Pres, $Head, $Alti, $Pitch, $Roll, $PosFlag, $PresFlag, $HeadFlag, $AltiFlag, $AttitFlag) = split (',', $raw); # # Read edited record into my variables # my (undef, undef, undef, $eUsec, $eLat, $eLon, undef, undef, $ePres, undef, undef, undef, undef, undef, undef, undef, undef, undef) = split (',', $edited); ##print "Lat diff = ", $Lat - $eLat, " Lon diff = ", $Lon - $eLon, "\n"; push @latDiff, $Lat - $eLat; push @lonDiff, $Lon - $eLon; } # End while ( # # Compute stats on pos differences # undef $lonStats; undef $latStats; my $lonStats = Statistics::Descriptive::Full->new(); my $latStats = Statistics::Descriptive::Full->new(); $lonStats->add_data(@lonDiff); $latStats->add_data(@latDiff); if ( $outputExcel ) { printf "\t %9.6f\n", ( sqrt($lonStats->variance) + sqrt($latStats->variance) ) / 2; } else { printf " lonDiff SD = %9.6f latDiff SD = %9.6f, Ave = %9.6f\n", sqrt($lonStats->variance), sqrt($latStats->variance), (sqrt($lonStats->variance) + sqrt($latStats->variance) ) / 2; } } # End calcSD() {noformat}]]> 1179928 1212694 1179929 5407179 5374443 1441793 2007-06-08 14:32:57.067 2009-02-05 21:43:36.160 1212692 [#frozen Pos] [excel]\n\n" unless -d $ARGV[0]; $showOver = $ARGV[1] || 60; # Show if pos repeats for more times than this $outputExcel = 0; $outputExcel = 1 if ( $ARGV[2] =~ /excel/i ); $| = 1; $deltaT = 5 / 60; # # Open directory and get list of all the raw nav files in it. Handle vnta & tibr # opendir (NAVDIR, $ARGV[0]) || die "Can't open directory $ARGV[0]: $!\n"; if ( $ARGV[0] =~ /vnta/ ) { @navfiles = grep { /vnta.txt$/ } readdir (NAVDIR); } elsif ( $ARGV[0] =~ /tibr/ ) { @navfiles = grep { /tibr.txt$/ } readdir (NAVDIR); } else { die "$0 works only for vnta or tibr directories.\n"; } die "Empty \@navFiles list. Was reading from: $ARGV[0]\n" unless @navfiles; closedir (NAVDIR); # # Print header for output # - the aveSD column is the average standard deviation between the original and edited nav # if the values are 0.00 then the original and edited data are identical. Larger # values will indicate a degraded performance in the nav system. There will natuarlly # be a correlation with performance accuracy and depth. # if ( $outputExcel ) { ##print "File,esecs,Lat,Lon,maxPres,numFreezes,totalDuration,recordDuration,aveSD,editedData\n"; print "File\tesecs\tLat\tLon\tmaxPres\tnumFreezes\ttotalDuration\trecordDuration\taveSD\teditedData\n"; } else { print "File str (hhmmss) end froz Lat froz lon sPres ePres # duration\n"; } # # Loop through all the files and collect stats on frozen positions # foreach ( @navfiles ) { $file = $_; print "file = $file\n" if $debug; if ( $outputExcel ) { } else { print "$file\n"; } calcFreezes(); calcSD(); } # End foreach #------------------------------------ # Subroutines: just to partition the work. No args passed. Use global vars # sub calcFreezes { open (NAVFILE, "$ARGV[0]/${file}") || die "Can't open file $ARGV[0]/${file}: $!\n"; $oldLat = 0; $oldLon = 0; $sameCount = 0; $notSame = 0; $prFile = $file; # Print file string, for listing below $prFile =~ s/vnta.txt//; $timeSum = 0; $recSum = 0; $numFrozen = 0; $maxPres = 0; while ( ) { next unless /^\d/; # Skip rec unless it begins with number # # Read record into my variables # ($Year, $Day, $Time, $Usec, $Lat, $Lon, $East, $North, $Pres, $Head, $Alti, $Pitch, $Roll, $PosFlag, $PresFlag, $HeadFlag, $AltiFlag, $AttitFlag) = split (',', $_); if ( $maxPres < $Pres ) { $maxPres = $Pres; } # # Count records of unchanging Lat & Lon # print "$_\n" if $debug; if ( $Lat == $oldLat && $Lon == $oldLon ) { $sameCount++; if ( $sameCount == 1 ) { $begUsec = $Usec; $begTime = $Time; $begPres = $Pres; } } elsif ( $oldLat != 0 ) { $notSame = 1; $timeDiffSec = $Usec - $begUsec; $duration = sprintf("%6.1f min", $timeDiffSec / 60); } # End if ( $Lat ... # # Print out stats for frozen position after we've counted them # if ( $sameCount > $showOver && $notSame ) { if ( $outputExcel ) { } else { printf ("%10s %d to %d %f %f %5.1f to %5.1F %3d %s\n", $prFile, $begTime, $Time, $oldLat, $oldLon, $begPres, $Pres, $sameCount, $duration); $prFile = ' '; } $recSum += $deltaT * $sameCount; $sameCount = 0; $notSame = 0; $timeSum += $duration; $numFrozen++; ##die "Debugging\n";; } # # Reset counter if not over our $showOver criteria # elsif ( $sameCount <= $showOver && $notSame ) { $sameCount = 0; $notSame = 0; } # End if ( $sameCount > $oldLat = $Lat; $oldLon = $Lon; } # End while (NAVFILE # # Print summary for this day # if ( $outputExcel ) { ##print "$prFile,$Usec,$Lat,$Lon,$maxPres,$numFrozen,$timeSum,", sprintf("%6.f", $recSum); print "$prFile\t$Usec\t$Lat\t$Lon\t$maxPres\t$numFrozen\t$timeSum\t", sprintf("%6.f", $recSum); } else { if ( $numFrozen > 0 ) { printf (" %3d frozen positions over %3d records long lasting total of %5.1f minutes\n", $numFrozen, $showOver, $timeSum); } } # End if ( $outputExcel } # End calcFreezes() sub calcSD { # # Create list of lat & lon of differences for the day # open (NAVFILE, "$ARGV[0]/${file}") || die "Can't open file $ARGV[0]/${file}: $!\n"; $editedFile = $file; $editedFile =~ s/\.txt/edited.txt/; if ( -f "$ARGV[0]/${editedFile}" ) { open (EDITEDNAV, "$ARGV[0]/${editedFile}") || die "Can't open file $ARGV[0]/${editedFile}: $!\n"; } else { print STDERR "Can't open file $ARGV[0]/${editedFile}: $!\n"; print STDERR "Skipping this file.\n"; if ( $outputExcel ) { ##print ",0,NoEditedNav\n"; print "\t0\tNoEditedNav\n"; } return; } undef @lonDiff; undef @latDiff; while ( ($raw = ) && ($edited = ) ) { next unless $raw =~ /^\d/; # Skip rec unless it begins with number # # Read raw record into my variables # my ($Year, $Day, $Time, $Usec, $Lat, $Lon, $East, $North, $Pres, $Head, $Alti, $Pitch, $Roll, $PosFlag, $PresFlag, $HeadFlag, $AltiFlag, $AttitFlag) = split (',', $raw); # # Read edited record into my variables # my (undef, undef, undef, $eUsec, $eLat, $eLon, undef, undef, $ePres, undef, undef, undef, undef, undef, undef, undef, undef, undef) = split (',', $edited); ##print "Lat diff = ", $Lat - $eLat, " Lon diff = ", $Lon - $eLon, "\n"; push @latDiff, $Lat - $eLat; push @lonDiff, $Lon - $eLon; } # End while ( # # Compute stats on pos differences # undef $lonStats; undef $latStats; my $lonStats = Statistics::Descriptive::Full->new(); my $latStats = Statistics::Descriptive::Full->new(); $lonStats->add_data(@lonDiff); $latStats->add_data(@latDiff); if ( $outputExcel ) { printf "\t %9.6f\n", ( sqrt($lonStats->variance) + sqrt($latStats->variance) ) / 2; } else { printf " lonDiff SD = %9.6f latDiff SD = %9.6f, Ave = %9.6f\n", sqrt($lonStats->variance), sqrt($latStats->variance), (sqrt($lonStats->variance) + sqrt($latStats->variance) ) / 2; } } # End calcSD() {noformat}]]> 1179927 1212697 [#frozen Pos] [excel]\n\n" unless -d $ARGV[0]; $showOver = $ARGV[1] || 60; # Show if pos repeats for more times than this $outputExcel = 0; $outputExcel = 1 if ( $ARGV[2] =~ /excel/i ); $| = 1; $deltaT = 5 / 60; # # Open directory and get list of all the raw nav files in it. Handle vnta & tibr # opendir (NAVDIR, $ARGV[0]) || die "Can't open directory $ARGV[0]: $!\n"; if ( $ARGV[0] =~ /vnta/ ) { @navfiles = grep { /vnta.txt$/ } readdir (NAVDIR); } elsif ( $ARGV[0] =~ /tibr/ ) { @navfiles = grep { /tibr.txt$/ } readdir (NAVDIR); } else { die "$0 works only for vnta or tibr directories.\n"; } die "Empty \@navFiles list. Was reading from: $ARGV[0]\n" unless @navfiles; closedir (NAVDIR); # # Print header for output # - the aveSD column is the average standard deviation between the original and edited nav # if the values are 0.00 then the original and edited data are identical. Larger # values will indicate a degraded performance in the nav system. There will natuarlly # be a correlation with performance accuracy and depth. # if ( $outputExcel ) { ##print "File,esecs,Lat,Lon,maxPres,numFreezes,totalDuration,recordDuration,aveSD,editedData\n"; print "File\tesecs\tLat\tLon\tmaxPres\tnumFreezes\ttotalDuration\trecordDuration\taveSD\teditedData\n"; } else { print "File str (hhmmss) end froz Lat froz lon sPres ePres # duration\n"; } # # Loop through all the files and collect stats on frozen positions # foreach ( @navfiles ) { $file = $_; print "file = $file\n" if $debug; if ( $outputExcel ) { } else { print "$file\n"; } calcFreezes(); calcSD(); } # End foreach #------------------------------------ # Subroutines: just to partition the work. No args passed. Use global vars # sub calcFreezes { open (NAVFILE, "$ARGV[0]/${file}") || die "Can't open file $ARGV[0]/${file}: $!\n"; $oldLat = 0; $oldLon = 0; $sameCount = 0; $notSame = 0; $prFile = $file; # Print file string, for listing below $prFile =~ s/vnta.txt//; $timeSum = 0; $recSum = 0; $numFrozen = 0; $maxPres = 0; while ( ) { next unless /^\d/; # Skip rec unless it begins with number # # Read record into my variables # ($Year, $Day, $Time, $Usec, $Lat, $Lon, $East, $North, $Pres, $Head, $Alti, $Pitch, $Roll, $PosFlag, $PresFlag, $HeadFlag, $AltiFlag, $AttitFlag) = split (',', $_); if ( $maxPres < $Pres ) { $maxPres = $Pres; } # # Count records of unchanging Lat & Lon # print "$_\n" if $debug; if ( $Lat == $oldLat && $Lon == $oldLon ) { $sameCount++; if ( $sameCount == 1 ) { $begUsec = $Usec; $begTime = $Time; $begPres = $Pres; } } elsif ( $oldLat != 0 ) { $notSame = 1; $timeDiffSec = $Usec - $begUsec; $duration = sprintf("%6.1f min", $timeDiffSec / 60); } # End if ( $Lat ... # # Print out stats for frozen position after we've counted them # if ( $sameCount > $showOver && $notSame ) { if ( $outputExcel ) { } else { printf ("%10s %d to %d %f %f %5.1f to %5.1F %3d %s\n", $prFile, $begTime, $Time, $oldLat, $oldLon, $begPres, $Pres, $sameCount, $duration); $prFile = ' '; } $recSum += $deltaT * $sameCount; $sameCount = 0; $notSame = 0; $timeSum += $duration; $numFrozen++; ##die "Debugging\n";; } # # Reset counter if not over our $showOver criteria # elsif ( $sameCount <= $showOver && $notSame ) { $sameCount = 0; $notSame = 0; } # End if ( $sameCount > $oldLat = $Lat; $oldLon = $Lon; } # End while (NAVFILE # # Print summary for this day # if ( $outputExcel ) { ##print "$prFile,$Usec,$Lat,$Lon,$maxPres,$numFrozen,$timeSum,", sprintf("%6.f", $recSum); print "$prFile\t$Usec\t$Lat\t$Lon\t$maxPres\t$numFrozen\t$timeSum\t", sprintf("%6.f", $recSum); } else { if ( $numFrozen > 0 ) { printf (" %3d frozen positions over %3d records long lasting total of %5.1f minutes\n", $numFrozen, $showOver, $timeSum); } } # End if ( $outputExcel } # End calcFreezes() sub calcSD { # # Create list of lat & lon of differences for the day # open (NAVFILE, "$ARGV[0]/${file}") || die "Can't open file $ARGV[0]/${file}: $!\n"; $editedFile = $file; $editedFile =~ s/\.txt/edited.txt/; if ( -f "$ARGV[0]/${editedFile}" ) { open (EDITEDNAV, "$ARGV[0]/${editedFile}") || die "Can't open file $ARGV[0]/${editedFile}: $!\n"; } else { print STDERR "Can't open file $ARGV[0]/${editedFile}: $!\n"; print STDERR "Skipping this file.\n"; if ( $outputExcel ) { ##print ",0,NoEditedNav\n"; print "\t0\tNoEditedNav\n"; } return; } undef @lonDiff; undef @latDiff; while ( ($raw = ) && ($edited = ) ) { next unless $raw =~ /^\d/; # Skip rec unless it begins with number # # Read raw record into my variables # my ($Year, $Day, $Time, $Usec, $Lat, $Lon, $East, $North, $Pres, $Head, $Alti, $Pitch, $Roll, $PosFlag, $PresFlag, $HeadFlag, $AltiFlag, $AttitFlag) = split (',', $raw); # # Read edited record into my variables # my (undef, undef, undef, $eUsec, $eLat, $eLon, undef, undef, $ePres, undef, undef, undef, undef, undef, undef, undef, undef, undef) = split (',', $edited); ##print "Lat diff = ", $Lat - $eLat, " Lon diff = ", $Lon - $eLon, "\n"; push @latDiff, $Lat - $eLat; push @lonDiff, $Lon - $eLon; } # End while ( # # Compute stats on pos differences # undef $lonStats; undef $latStats; my $lonStats = Statistics::Descriptive::Full->new(); my $latStats = Statistics::Descriptive::Full->new(); $lonStats->add_data(@lonDiff); $latStats->add_data(@latDiff); if ( $outputExcel ) { printf "\t %9.6f\n", ( sqrt($lonStats->variance) + sqrt($latStats->variance) ) / 2; } else { printf " lonDiff SD = %9.6f latDiff SD = %9.6f, Ave = %9.6f\n", sqrt($lonStats->variance), sqrt($latStats->variance), (sqrt($lonStats->variance) + sqrt($latStats->variance) ) / 2; } } # End calcSD() {noformat}]]> 1179932 1212698 [#frozen Pos] [excel]\n\n" unless -d $ARGV[0]; $showOver = $ARGV[1] || 60; # Show if pos repeats for more times than this $outputExcel = 0; $outputExcel = 1 if ( $ARGV[2] =~ /excel/i ); $| = 1; $deltaT = 5 / 60; # # Open directory and get list of all the raw nav files in it. Handle vnta & tibr # opendir (NAVDIR, $ARGV[0]) || die "Can't open directory $ARGV[0]: $!\n"; if ( $ARGV[0] =~ /vnta/ ) { @navfiles = grep { /vnta.txt$/ } readdir (NAVDIR); } elsif ( $ARGV[0] =~ /tibr/ ) { @navfiles = grep { /tibr.txt$/ } readdir (NAVDIR); } else { die "$0 works only for vnta or tibr directories.\n"; } die "Empty \@navFiles list. Was reading from: $ARGV[0]\n" unless @navfiles; closedir (NAVDIR); # # Print header for output # - the aveSD column is the average standard deviation between the original and edited nav # if the values are 0.00 then the original and edited data are identical. Larger # values will indicate a degraded performance in the nav system. There will natuarlly # be a correlation with performance accuracy and depth. # if ( $outputExcel ) { ##print "File,esecs,Lat,Lon,maxPres,numFreezes,totalDuration,recordDuration,aveSD,editedData\n"; print "File\tesecs\tLat\tLon\tmaxPres\tnumFreezes\ttotalDuration\trecordDuration\taveSD\teditedData\n"; } else { print "File str (hhmmss) end froz Lat froz lon sPres ePres # duration\n"; } # # Loop through all the files and collect stats on frozen positions # foreach ( @navfiles ) { $file = $_; print "file = $file\n" if $debug; if ( $outputExcel ) { } else { print "$file\n"; } calcFreezes(); calcSD(); } # End foreach #------------------------------------ # Subroutines: just to partition the work. No args passed. Use global vars # sub calcFreezes { open (NAVFILE, "$ARGV[0]/${file}") || die "Can't open file $ARGV[0]/${file}: $!\n"; $oldLat = 0; $oldLon = 0; $sameCount = 0; $notSame = 0; $prFile = $file; # Print file string, for listing below $prFile =~ s/vnta.txt//; $timeSum = 0; $recSum = 0; $numFrozen = 0; $maxPres = 0; while ( ) { next unless /^\d/; # Skip rec unless it begins with number # # Read record into my variables # ($Year, $Day, $Time, $Usec, $Lat, $Lon, $East, $North, $Pres, $Head, $Alti, $Pitch, $Roll, $PosFlag, $PresFlag, $HeadFlag, $AltiFlag, $AttitFlag) = split (',', $_); if ( $maxPres < $Pres ) { $maxPres = $Pres; } # # Count records of unchanging Lat & Lon # print "$_\n" if $debug; if ( $Lat == $oldLat && $Lon == $oldLon ) { $sameCount++; if ( $sameCount == 1 ) { $begUsec = $Usec; $begTime = $Time; $begPres = $Pres; } } elsif ( $oldLat != 0 ) { $notSame = 1; $timeDiffSec = $Usec - $begUsec; $duration = sprintf("%6.1f min", $timeDiffSec / 60); } # End if ( $Lat ... # # Print out stats for frozen position after we've counted them # if ( $sameCount > $showOver && $notSame ) { if ( $outputExcel ) { } else { printf ("%10s %d to %d %f %f %5.1f to %5.1F %3d %s\n", $prFile, $begTime, $Time, $oldLat, $oldLon, $begPres, $Pres, $sameCount, $duration); $prFile = ' '; } $recSum += $deltaT * $sameCount; $sameCount = 0; $notSame = 0; $timeSum += $duration; $numFrozen++; ##die "Debugging\n";; } # # Reset counter if not over our $showOver criteria # elsif ( $sameCount <= $showOver && $notSame ) { $sameCount = 0; $notSame = 0; } # End if ( $sameCount > $oldLat = $Lat; $oldLon = $Lon; } # End while (NAVFILE # # Print summary for this day # if ( $outputExcel ) { ##print "$prFile,$Usec,$Lat,$Lon,$maxPres,$numFrozen,$timeSum,", sprintf("%6.f", $recSum); print "$prFile\t$Usec\t$Lat\t$Lon\t$maxPres\t$numFrozen\t$timeSum\t", sprintf("%6.f", $recSum); } else { if ( $numFrozen > 0 ) { printf (" %3d frozen positions over %3d records long lasting total of %5.1f minutes\n", $numFrozen, $showOver, $timeSum); } } # End if ( $outputExcel } # End calcFreezes() sub calcSD { # # Create list of lat & lon of differences for the day # open (NAVFILE, "$ARGV[0]/${file}") || die "Can't open file $ARGV[0]/${file}: $!\n"; $editedFile = $file; $editedFile =~ s/\.txt/edited.txt/; if ( -f "$ARGV[0]/${editedFile}" ) { open (EDITEDNAV, "$ARGV[0]/${editedFile}") || die "Can't open file $ARGV[0]/${editedFile}: $!\n"; } else { print STDERR "Can't open file $ARGV[0]/${editedFile}: $!\n"; print STDERR "Skipping this file.\n"; if ( $outputExcel ) { ##print ",0,NoEditedNav\n"; print "\t0\tNoEditedNav\n"; } return; } undef @lonDiff; undef @latDiff; while ( ($raw = ) && ($edited = ) ) { next unless $raw =~ /^\d/; # Skip rec unless it begins with number # # Read raw record into my variables # my ($Year, $Day, $Time, $Usec, $Lat, $Lon, $East, $North, $Pres, $Head, $Alti, $Pitch, $Roll, $PosFlag, $PresFlag, $HeadFlag, $AltiFlag, $AttitFlag) = split (',', $raw); # # Read edited record into my variables # my (undef, undef, undef, $eUsec, $eLat, $eLon, undef, undef, $ePres, undef, undef, undef, undef, undef, undef, undef, undef, undef) = split (',', $edited); ##print "Lat diff = ", $Lat - $eLat, " Lon diff = ", $Lon - $eLon, "\n"; push @latDiff, $Lat - $eLat; push @lonDiff, $Lon - $eLon; } # End while ( # # Compute stats on pos differences # undef $lonStats; undef $latStats; my $lonStats = Statistics::Descriptive::Full->new(); my $latStats = Statistics::Descriptive::Full->new(); $lonStats->add_data(@lonDiff); $latStats->add_data(@latDiff); if ( $outputExcel ) { printf "\t %9.6f\n", ( sqrt($lonStats->variance) + sqrt($latStats->variance) ) / 2; } else { printf " lonDiff SD = %9.6f latDiff SD = %9.6f, Ave = %9.6f\n", sqrt($lonStats->variance), sqrt($latStats->variance), (sqrt($lonStats->variance) + sqrt($latStats->variance) ) / 2; } } # End calcSD() {noformat}]]> 1179933 1212695 1179930 1212696 [#frozen Pos] [excel]\n\n" unless -d $ARGV[0]; $showOver = $ARGV[1] || 60; # Show if pos repeats for more times than this $outputExcel = 0; $outputExcel = 1 if ( $ARGV[2] =~ /excel/i ); $| = 1; $deltaT = 5 / 60; # # Open directory and get list of all the raw nav files in it. Handle vnta & tibr # opendir (NAVDIR, $ARGV[0]) || die "Can't open directory $ARGV[0]: $!\n"; if ( $ARGV[0] =~ /vnta/ ) { @navfiles = grep { /vnta.txt$/ } readdir (NAVDIR); } elsif ( $ARGV[0] =~ /tibr/ ) { @navfiles = grep { /tibr.txt$/ } readdir (NAVDIR); } else { die "$0 works only for vnta or tibr directories.\n"; } die "Empty \@navFiles list. Was reading from: $ARGV[0]\n" unless @navfiles; closedir (NAVDIR); # # Print header for output # - the aveSD column is the average standard deviation between the original and edited nav # if the values are 0.00 then the original and edited data are identical. Larger # values will indicate a degraded performance in the nav system. There will natuarlly # be a correlation with performance accuracy and depth. # if ( $outputExcel ) { ##print "File,esecs,Lat,Lon,maxPres,numFreezes,totalDuration,recordDuration,aveSD,editedData\n"; print "File\tesecs\tLat\tLon\tmaxPres\tnumFreezes\ttotalDuration\trecordDuration\taveSD\teditedData\n"; } else { print "File str (hhmmss) end froz Lat froz lon sPres ePres # duration\n"; } # # Loop through all the files and collect stats on frozen positions # foreach ( @navfiles ) { $file = $_; print "file = $file\n" if $debug; if ( $outputExcel ) { } else { print "$file\n"; } calcFreezes(); calcSD(); } # End foreach #------------------------------------ # Subroutines: just to partition the work. No args passed. Use global vars # sub calcFreezes { open (NAVFILE, "$ARGV[0]/${file}") || die "Can't open file $ARGV[0]/${file}: $!\n"; $oldLat = 0; $oldLon = 0; $sameCount = 0; $notSame = 0; $prFile = $file; # Print file string, for listing below $prFile =~ s/vnta.txt//; $timeSum = 0; $recSum = 0; $numFrozen = 0; $maxPres = 0; while ( ) { next unless /^\d/; # Skip rec unless it begins with number # # Read record into my variables # ($Year, $Day, $Time, $Usec, $Lat, $Lon, $East, $North, $Pres, $Head, $Alti, $Pitch, $Roll, $PosFlag, $PresFlag, $HeadFlag, $AltiFlag, $AttitFlag) = split (',', $_); if ( $maxPres < $Pres ) { $maxPres = $Pres; } # # Count records of unchanging Lat & Lon # print "$_\n" if $debug; if ( $Lat == $oldLat && $Lon == $oldLon ) { $sameCount++; if ( $sameCount == 1 ) { $begUsec = $Usec; $begTime = $Time; $begPres = $Pres; } } elsif ( $oldLat != 0 ) { $notSame = 1; $timeDiffSec = $Usec - $begUsec; $duration = sprintf("%6.1f min", $timeDiffSec / 60); } # End if ( $Lat ... # # Print out stats for frozen position after we've counted them # if ( $sameCount > $showOver && $notSame ) { if ( $outputExcel ) { } else { printf ("%10s %d to %d %f %f %5.1f to %5.1F %3d %s\n", $prFile, $begTime, $Time, $oldLat, $oldLon, $begPres, $Pres, $sameCount, $duration); $prFile = ' '; } $recSum += $deltaT * $sameCount; $sameCount = 0; $notSame = 0; $timeSum += $duration; $numFrozen++; ##die "Debugging\n";; } # # Reset counter if not over our $showOver criteria # elsif ( $sameCount <= $showOver && $notSame ) { $sameCount = 0; $notSame = 0; } # End if ( $sameCount > $oldLat = $Lat; $oldLon = $Lon; } # End while (NAVFILE # # Print summary for this day # if ( $outputExcel ) { ##print "$prFile,$Usec,$Lat,$Lon,$maxPres,$numFrozen,$timeSum,", sprintf("%6.f", $recSum); print "$prFile\t$Usec\t$Lat\t$Lon\t$maxPres\t$numFrozen\t$timeSum\t", sprintf("%6.f", $recSum); } else { if ( $numFrozen > 0 ) { printf (" %3d frozen positions over %3d records long lasting total of %5.1f minutes\n", $numFrozen, $showOver, $timeSum); } } # End if ( $outputExcel } # End calcFreezes() sub calcSD { # # Create list of lat & lon of differences for the day # open (NAVFILE, "$ARGV[0]/${file}") || die "Can't open file $ARGV[0]/${file}: $!\n"; $editedFile = $file; $editedFile =~ s/\.txt/edited.txt/; if ( -f "$ARGV[0]/${editedFile}" ) { open (EDITEDNAV, "$ARGV[0]/${editedFile}") || die "Can't open file $ARGV[0]/${editedFile}: $!\n"; } else { print STDERR "Can't open file $ARGV[0]/${editedFile}: $!\n"; print STDERR "Skipping this file.\n"; if ( $outputExcel ) { ##print ",0,NoEditedNav\n"; print "\t0\tNoEditedNav\n"; } return; } undef @lonDiff; undef @latDiff; while ( ($raw = ) && ($edited = ) ) { next unless $raw =~ /^\d/; # Skip rec unless it begins with number # # Read raw record into my variables # my ($Year, $Day, $Time, $Usec, $Lat, $Lon, $East, $North, $Pres, $Head, $Alti, $Pitch, $Roll, $PosFlag, $PresFlag, $HeadFlag, $AltiFlag, $AttitFlag) = split (',', $raw); # # Read edited record into my variables # my (undef, undef, undef, $eUsec, $eLat, $eLon, undef, undef, $ePres, undef, undef, undef, undef, undef, undef, undef, undef, undef) = split (',', $edited); ##print "Lat diff = ", $Lat - $eLat, " Lon diff = ", $Lon - $eLon, "\n"; push @latDiff, $Lat - $eLat; push @lonDiff, $Lon - $eLon; } # End while ( # # Compute stats on pos differences # undef $lonStats; undef $latStats; my $lonStats = Statistics::Descriptive::Full->new(); my $latStats = Statistics::Descriptive::Full->new(); $lonStats->add_data(@lonDiff); $latStats->add_data(@latDiff); if ( $outputExcel ) { printf "\t %9.6f\n", ( sqrt($lonStats->variance) + sqrt($latStats->variance) ) / 2; } else { printf " lonDiff SD = %9.6f latDiff SD = %9.6f, Ave = %9.6f\n", sqrt($lonStats->variance), sqrt($latStats->variance), (sqrt($lonStats->variance) + sqrt($latStats->variance) ) / 2; } } # End calcSD() {noformat}]]> 1179931 1212701 1179936 32440323 32407555 1212702 1179937 1212699 [#frozen Pos] [excel]\n\n" unless -d $ARGV[0]; $showOver = $ARGV[1] || 60; # Show if pos repeats for more times than this $outputExcel = 0; $outputExcel = 1 if ( $ARGV[2] =~ /excel/i ); $| = 1; $deltaT = 5 / 60; # # Open directory and get list of all the raw nav files in it. Handle vnta & tibr # opendir (NAVDIR, $ARGV[0]) || die "Can't open directory $ARGV[0]: $!\n"; if ( $ARGV[0] =~ /vnta/ ) { @navfiles = grep { /vnta.txt$/ } readdir (NAVDIR); } elsif ( $ARGV[0] =~ /tibr/ ) { @navfiles = grep { /tibr.txt$/ } readdir (NAVDIR); } else { die "$0 works only for vnta or tibr directories.\n"; } die "Empty \@navFiles list. Was reading from: $ARGV[0]\n" unless @navfiles; closedir (NAVDIR); # # Print header for output # - the aveSD column is the average standard deviation between the original and edited nav # if the values are 0.00 then the original and edited data are identical. Larger # values will indicate a degraded performance in the nav system. There will natuarlly # be a correlation with performance accuracy and depth. # if ( $outputExcel ) { ##print "File,esecs,Lat,Lon,maxPres,numFreezes,totalDuration,recordDuration,aveSD,editedData\n"; print "File\tesecs\tLat\tLon\tmaxPres\tnumFreezes\ttotalDuration\trecordDuration\taveSD\teditedData\n"; } else { print "File str (hhmmss) end froz Lat froz lon sPres ePres # duration\n"; } # # Loop through all the files and collect stats on frozen positions # foreach ( @navfiles ) { $file = $_; print "file = $file\n" if $debug; if ( $outputExcel ) { } else { print "$file\n"; } calcFreezes(); calcSD(); } # End foreach #------------------------------------ # Subroutines: just to partition the work. No args passed. Use global vars # sub calcFreezes { open (NAVFILE, "$ARGV[0]/${file}") || die "Can't open file $ARGV[0]/${file}: $!\n"; $oldLat = 0; $oldLon = 0; $sameCount = 0; $notSame = 0; $prFile = $file; # Print file string, for listing below $prFile =~ s/vnta.txt//; $timeSum = 0; $recSum = 0; $numFrozen = 0; $maxPres = 0; while ( ) { next unless /^\d/; # Skip rec unless it begins with number # # Read record into my variables # ($Year, $Day, $Time, $Usec, $Lat, $Lon, $East, $North, $Pres, $Head, $Alti, $Pitch, $Roll, $PosFlag, $PresFlag, $HeadFlag, $AltiFlag, $AttitFlag) = split (',', $_); if ( $maxPres < $Pres ) { $maxPres = $Pres; } # # Count records of unchanging Lat & Lon # print "$_\n" if $debug; if ( $Lat == $oldLat && $Lon == $oldLon ) { $sameCount++; if ( $sameCount == 1 ) { $begUsec = $Usec; $begTime = $Time; $begPres = $Pres; } } elsif ( $oldLat != 0 ) { $notSame = 1; $timeDiffSec = $Usec - $begUsec; $duration = sprintf("%6.1f min", $timeDiffSec / 60); } # End if ( $Lat ... # # Print out stats for frozen position after we've counted them # if ( $sameCount > $showOver && $notSame ) { if ( $outputExcel ) { } else { printf ("%10s %d to %d %f %f %5.1f to %5.1F %3d %s\n", $prFile, $begTime, $Time, $oldLat, $oldLon, $begPres, $Pres, $sameCount, $duration); $prFile = ' '; } $recSum += $deltaT * $sameCount; $sameCount = 0; $notSame = 0; $timeSum += $duration; $numFrozen++; ##die "Debugging\n";; } # # Reset counter if not over our $showOver criteria # elsif ( $sameCount <= $showOver && $notSame ) { $sameCount = 0; $notSame = 0; } # End if ( $sameCount > $oldLat = $Lat; $oldLon = $Lon; } # End while (NAVFILE # # Print summary for this day # if ( $outputExcel ) { ##print "$prFile,$Usec,$Lat,$Lon,$maxPres,$numFrozen,$timeSum,", sprintf("%6.f", $recSum); print "$prFile\t$Usec\t$Lat\t$Lon\t$maxPres\t$numFrozen\t$timeSum\t", sprintf("%6.f", $recSum); } else { if ( $numFrozen > 0 ) { printf (" %3d frozen positions over %3d records long lasting total of %5.1f minutes\n", $numFrozen, $showOver, $timeSum); } } # End if ( $outputExcel } # End calcFreezes() sub calcSD { # # Create list of lat & lon of differences for the day # open (NAVFILE, "$ARGV[0]/${file}") || die "Can't open file $ARGV[0]/${file}: $!\n"; $editedFile = $file; $editedFile =~ s/\.txt/edited.txt/; if ( -f "$ARGV[0]/${editedFile}" ) { open (EDITEDNAV, "$ARGV[0]/${editedFile}") || die "Can't open file $ARGV[0]/${editedFile}: $!\n"; } else { print STDERR "Can't open file $ARGV[0]/${editedFile}: $!\n"; print STDERR "Skipping this file.\n"; if ( $outputExcel ) { ##print ",0,NoEditedNav\n"; print "\t0\tNoEditedNav\n"; } return; } undef @lonDiff; undef @latDiff; while ( ($raw = ) && ($edited = ) ) { next unless $raw =~ /^\d/; # Skip rec unless it begins with number # # Read raw record into my variables # my ($Year, $Day, $Time, $Usec, $Lat, $Lon, $East, $North, $Pres, $Head, $Alti, $Pitch, $Roll, $PosFlag, $PresFlag, $HeadFlag, $AltiFlag, $AttitFlag) = split (',', $raw); # # Read edited record into my variables # my (undef, undef, undef, $eUsec, $eLat, $eLon, undef, undef, $ePres, undef, undef, undef, undef, undef, undef, undef, undef, undef) = split (',', $edited); ##print "Lat diff = ", $Lat - $eLat, " Lon diff = ", $Lon - $eLon, "\n"; push @latDiff, $Lat - $eLat; push @lonDiff, $Lon - $eLon; } # End while ( # # Compute stats on pos differences # undef $lonStats; undef $latStats; my $lonStats = Statistics::Descriptive::Full->new(); my $latStats = Statistics::Descriptive::Full->new(); $lonStats->add_data(@lonDiff); $latStats->add_data(@latDiff); if ( $outputExcel ) { printf "\t %9.6f\n", ( sqrt($lonStats->variance) + sqrt($latStats->variance) ) / 2; } else { printf " lonDiff SD = %9.6f latDiff SD = %9.6f, Ave = %9.6f\n", sqrt($lonStats->variance), sqrt($latStats->variance), (sqrt($lonStats->variance) + sqrt($latStats->variance) ) / 2; } } # End calcSD() {noformat}]]> 1179934 29851879 29819111 1212700 1179935 1212706 1179941 5407175 5374439 1212704 1179939 29851905 cd /mbari/Tempbox/mccann <103 elvis.shore.mbari.org /Tempbox/mccann> tar cvf 2012.268.07.tar 2012.268.07 2012.268.07/ 2012.268.07/SmartSamplerDataLog.log 2012.268.07/Gulper.log ... 2012.268.07/ctdDriver2.cfg 2012.268.07/gps.log 2012.268.07/m3dmgx1.log <105 elvis.shore.mbari.org /Tempbox/mccann> gzip 2012.268.07.tar {noformat} # Copy tarball to /usr/local/auv_ctd and run the auvportal which submits the data to SSDS: {noformat} <255 elvis.shore.mbari.org /Tempbox/mccann> cp 2012.268.07.tar.gz /usr/local/auv_ctd <257 elvis.shore.mbari.org /local/auv_ctd> ./portal AUVCTD2012268 2012.268.07.tar.gz Tue, 25 Sep 2012 09:27:43 PDT: processConnection() got message: AUVCTD2012268;2012.268.07.tar.gz dstDir.getName() = 2012268 Unpacking /usr/local/auv_ctd/2012.268.07.tar.gz Copying /usr/local/auv_ctd/2012268/2012.268.07/ctdDriver2.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/ctdDriver2.log Copying /usr/local/auv_ctd/2012268/2012.268.07/gps.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/gps.log Copying /usr/local/auv_ctd/2012268/2012.268.07/isuslog.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/isuslog.log Copying /usr/local/auv_ctd/2012268/2012.268.07/devices.cfg to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/devices.cfg ... DEBUG [Thread-0] (XmlBuilder.java:538) - appendPrimitiveType called with method name isValidRole on object of class moos.ssds.metadata.DataProducer DEBUG [Thread-0] (XmlBuilder.java:482) - appendElement called on class moos.ssds.metadata.DataProducer using method getClass DEBUG [Thread-0] (XmlBuilder.java:278) - Converting java.lang.Class(class moos.ssds.metadata.DataProducer) to XML Marshalled the AUV mission in /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 into XML Attempting to publish AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ published XML describing mission AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ If everything goes well in SSDS:Ingest/Ruminate an email message to auvctd@listserver.mbari.org will arrive shortly. Tue, 25 Sep 2012 09:34:29 PDT: Done receiving AUV mission data. {noformat} Note: With the move of the portal to from jeckel to elvis the mailing to auvctd@listserver.mbari.org has stopped working. This needs to be debugged and fixed. # Process the LOPC data, e.g.: {noformat} <109 elvis.shore.mbari.org /u/ssdsadmin> cd ~/dev/auv_ctd/src/py/auv/ <110 elvis.shore.mbari.org /py/auv> processLOPC.pl 2013.224.02 {noformat} # Once the mission's instrument netcdf files have been created (check pismo:/opt/ssds/logs/ssds-updateBot.log for successful processing) the overall Dorado science data processing script can be edited with the new survey to be processed: {noformat} <102 elvis.shore.mbari.org /u/ssdsadmin> cd dev/auv_ctd/src/matlab/doradosdp/process/ <103 elvis.shore.mbari.org /doradosdp/process> vi reprocess_surveys.m {noformat} # Add a line for the new survey to the end of the reprocess_surveys.m file, commenting out the previous survey that was processed: {noformat} % CANON September 2012 - laps around ESP drifter - last one for this campaign %%AUVprocess_main('AUVCTD', 2012, ['258.00'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') % CANON September 2012 - laps around ESP drifter - last one for this campaign AUVprocess_main('AUVCTD', 2012, ['268.07'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') {noformat} # Execute the Matlab job with a shell script that properly sets the DISPLAY environment and executes reprocess_surveys.m with matlab: {noformat} <104 elvis.shore.mbari.org /doradosdp/process> RunReprocessJob {noformat} # Check for error messages. If everything works an email is sent to the notifyEmail setting in reprocess_surveys.m, usually auvctd@listserver.mbari.org. h2. Software checkout and repository code management The Software for both the AUV Portal and the Science data processing are located in the _auv_ctd_ module on the CVS server moonjelly.shore.mbari.org. To check the software to a system use your favorite CVS client, or on Unix use these commands: {noformat} cd cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). {noformat} Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): {noformat} {noformat} > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... {noformat} h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: {noformat} usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy {noformat} For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: {noformat} adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 {noformat} Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look something like this: {noformat} bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ {noformat} \\ On the portal computer the screen will display text something like this: {noformat} Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS {noformat} Note: Detailed command-line instructions for "portaling" AUVCTD mission data into SSDS are available in the README file in the project or in the operational location on jeckel.shore.mbari.org in /usr/local/auv_ctd/ directory. There is also the portal.log file there which logs the output of the deamon process running on jeckel that receives and processes the data from AUV uploads from the vehicle. The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: {noformat} > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> {noformat} So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: {noformat} Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds {noformat} +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables and an Ocean Data View spreadsheet file. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file). If one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files.]]> 29819155 31358982 31326214 31358980 31326212 31358984 31326216 10388335 10355607 1212690 1179925 5407216 5374480 5407217 5374481 5407203 5374467 5407227 5374491 1212707 1179942 29720642 29687875 21004738 20971975 5407234 5374499 5407219 5374483 5407221 5374485 5407223 5374487 10388300 10355572 5407226 5374490 5407225 5374489 21365769 21398481 4 2013-07-23 15:23:20.427 2013-08-02 09:50:41.427 21365583 33619970 Customize... menu (check the ArcMapNav box): \\ !ArcMabNav_fig1.png! Click the ArcMapNav Setup button (select the Ship and see that positions are being updated): !ArcMabNav_fig2.png!     Add the ROV and Ship Layers, and any other base layers you wish: !ArcMabNav_fig3.png!   Saving Markpoints (click on Mouse button or Target button): !ArcMabNav_fig4.png!\\ Getting Range and Bearing (click on Compass button): !ArcMabNav_fig5.png!  ]]> 33587202 21365722 21398437 3 2013-07-23 15:23:20.427 2013-08-02 09:46:44.440 21365583 21365721 21398436 2 2013-07-23 15:23:20.427 2013-07-23 15:24:13.137 21365583 5865474 5832706 10388378 10355650 31588354 31555586 31588353 31555585 21397976 21365245 21397983 21365252 21397984 21365253 21365740 21398452 1 2013-08-05 16:19:34.827 2013-08-05 16:19:34.827 21365738 21397979 21365248 21397981 21365250 22315069 22282301 8945918 8913160 8945917 8913159 8945922 8913165 8945920 8913163 9831042 9798289 21398038 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 h4. 16 May 2013 Another post to the pgsql-perform mail list asking for opinions on a more conventional server configuration: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 3:36:03 PM PDT To: pgsql-performance@postgresql.org Cc: Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On May 7, 2013, at 4:21 PM, Jeff Janes wrote: On Thu, May 2, 2013 at 6:35 PM, Scott Marlowe wrote: On Thu, May 2, 2013 at 5:11 PM, Mike McCann wrote: > Hello, > > We are in the fortunate situation of having more money than time to help > solve our PostgreSQL 9.1 performance problem. > > Our server hosts databases that are about 1 GB in size with the largest > tables having order 10 million 20-byte indexed records. The data are loaded > once and then read from a web app and other client programs. Some of the > queries execute ORDER BY on the results. There are typically less than a > dozen read-only concurrent connections to any one database. I wouldn't count on this being a problem that can be fixed merely by throwing money at it. How many rows does any one of these queries need to access and then ORDER BY? ... > > HP ProLiant DL360p Gen 8 > Dual Intel Xeon 2.4GHz 4-core E5-2609 CPUs > 64GB RAM > 2x146GB 15K SAS hard drives > 3x200GB SATA SLC SSDs > + the usual accessories (optical drive, rail kit, dual power supplies) If your DB is 1G, and will grow to 10G then the IO shouldn't be any problem, as the whole db should be cached in memory. But it can take a surprisingly long time to get it cached in the first place, from a cold start. If that is the problem, pg_prewarm could help. Cheers, Jeff Thank you everyone for your suggestions. It's clear that our current read performance was not limited by hardware. An 'explain analyze' for a sample query is: stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=541002.15..549456.68 rows=3381814 width=20) (actual time=6254.780..7244.074 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..354.385 rows=3381814 loops=1) Total runtime: 7425.854 ms (5 rows) Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? We have an alternate (a bit more conventional) server configuration that we are considering: HP ProLiant DL360p Gen 8 Dual Intel Xeon 3.3GHz 4-core E5-2643 CPUs 128GB PC3-12800 RAM 16x146GB 15K SAS hard drives HP Smart Array P822/2GB FBWC controller + P420i w/ 2GB FBWC + the usual accessories (optical drive, rail kit, dual power supplies) All suggestions welcomed! -Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} The responses: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 4:58:17 PM PDT To: Jeff Janes Cc: "pgsql-performance@postgresql.org" , Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden On May 13, 2013, at 4:24 PM, Jeff Janes wrote: > On Mon, May 13, 2013 at 3:36 PM, Mike McCann wrote: > > Increasing work_mem to 355 MB improves the performance by a factor of 2: > > stoqs_march2013_s=# set work_mem='355MB'; > SET > stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > > I tried changing random_page_cost to from 4 to 1 and saw no change. > > I'm wondering now what changes might get this query to run in less than one second. I think you are worrying about the wrong thing here. What is a web app going to do with 3,381,814 rows, once it obtains them? Your current testing is not testing the time it takes to stream that data to the client, or for the client to do something meaningful with that data. If you only plan to actually fetch a few dozen of those rows, then you probably need to incorporate that into your test, either by using a LIMIT, or by using a mock-up of the actual application to do some timings. Also, what is the type and collation of the column you are sorting on? non-'C' collations of text columns sort about 3 times slower than 'C' collation does. If all the data is in memory, then will faster CPU and memory be the things that help? Yes, those would help (it is not clear to me which of the two would help more), but I think you need to rethink your design of sending the entire database table to the application server for each page-view. Cheers, Jeff Hi Jeff, The datavalue column is double precision: stoqs_march2013_s=# \d+ stoqs_measuredparameter Table "public.stoqs_measuredparameter" Column | Type | Modifiers | Storage | Description ----------------+------------------+----------------------------------------------------------------------+---------+------------- id | integer | not null default nextval('stoqs_measuredparameter_id_seq'::regclass) | plain | measurement_id | integer | not null | plain | parameter_id | integer | not null | plain | datavalue | double precision | not null | plain | Indexes: "stoqs_measuredparameter_pkey" PRIMARY KEY, btree (id) "stoqs_measuredparameter_measurement_id_parameter_id_key" UNIQUE CONSTRAINT, btree (measurement_id, parameter_id) "stoqs_measuredparameter_datavalue" btree (datavalue) "stoqs_measuredparameter_measurement_id" btree (measurement_id) "stoqs_measuredparameter_parameter_id" btree (parameter_id) Foreign-key constraints: "stoqs_measuredparameter_measurement_id_fkey" FOREIGN KEY (measurement_id) REFERENCES stoqs_measurement(id) DEFERRABLE INITIALLY DEFERRED "stoqs_measuredparameter_parameter_id_fkey" FOREIGN KEY (parameter_id) REFERENCES stoqs_parameter(id) DEFERRABLE INITIALLY DEFERRED Has OIDs: no Thanks for the suggestion and advice to examine the web app performance. We've actually taken quite a few steps to optimize how the web app works. The example query I provided is a simple worst-case one that we can use to help us decide on the proper hardware. An actual query performed by the web app is: stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_march2013_s-# stoqs_parameter.name AS parameter__name, stoqs_march2013_s-# stoqs_parameter.standard_name AS parameter__standard_name, stoqs_march2013_s-# stoqs_measurement.depth AS measurement__depth, stoqs_march2013_s-# stoqs_measurement.geom AS measurement__geom, stoqs_march2013_s-# stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_march2013_s-# stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_march2013_s-# stoqs_measuredparameter.datavalue AS datavalue, stoqs_march2013_s-# stoqs_parameter.units AS parameter__units stoqs_march2013_s-# FROM stoqs_parameter p1, stoqs_march2013_s-# stoqs_measuredparameter stoqs_march2013_s-# INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) stoqs_march2013_s-# INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) stoqs_march2013_s-# INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) stoqs_march2013_s-# INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) stoqs_march2013_s-# INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) stoqs_march2013_s-# INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id stoqs_march2013_s-# WHERE (p1.name = 'sea_water_sigma_t') stoqs_march2013_s-# AND (mp1.datavalue > 25.19) stoqs_march2013_s-# AND (mp1.datavalue < 26.01) stoqs_march2013_s-# AND (mp1.parameter_id = p1.id) stoqs_march2013_s-# AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' stoqs_march2013_s(# AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' stoqs_march2013_s(# AND stoqs_parameter.name IN ('fl700_uncorr') stoqs_march2013_s(# AND stoqs_measurement.depth >= -1.88 stoqs_march2013_s(# AND stoqs_platform.name IN ('dorado') stoqs_march2013_s(# AND stoqs_measurement.depth <= 83.57) stoqs_march2013_s-# ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=770.211..770.211 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=770.200..770.200 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=770.036..770.036 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=91.369..680.072 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=91.346..577.633 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=91.280..531.408 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=91.202..116.140 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.002..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=91.194..109.846 rows=20170 loop s=1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=90.794..90.794 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=62.769..62.769 rows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=27.412..2 7.412 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.008..0. 017 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.012..0.012 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.007..0.010 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.004..0.004 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.004..0.004 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.151..0.151 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.035..0.145 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.005..0.069 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.014..0.014 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.008..0.012 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 770.445 ms (42 rows) We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Thanks, Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} Another response: {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 6:09:58 PM PDT To: Mike McCann Cc: Jeff Janes , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden On Mon, May 13, 2013 at 5:58 PM, Mike McCann wrote: > We assume that steps taken to improve the worst-case query scenario will > also improve these kind of queries. If anything above pops out as needing > better planning please let us know that too! Bad assumption. If your real workload will be queries like the one here that takes 700 ms, but you'll be running 10,000 of them a second, you're tuning / hardware choices are going to be much different then if your query is going to be the previous 7 second one. Use realistic queries, not ones that are nothing like what your real ones will be. then use pgbench and its ability to run custom sql scripts to get a REAL idea how your hardware performs. Note that if you will run the slow query you posted like once a minute and roll it up or cache it then don't get too worried about it. Pay attention to the queries that will add up, in aggregate, to your greatest load. {noformat} Not a whole lot of opinions except that tuning a server requires diligent work and performance is something one must do some work to achieve, and cannot be had by simply throwing money at the problem. h4. 20 May 2013 A few more responses over the weekend: {noformat} From: Greg Smith Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 7:44:16 PM PDT To: Mike McCann Cc: pgsql-performance@postgresql.org, Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On 5/13/13 6:36 PM, Mike McCann wrote: > stoqs_march2013_s=# explain analyze select * from > stoqs_measuredparameter order by datavalue; > > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual > time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 > rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > I tried changing random_page_cost to from 4 to 1 and saw no change. Have you tried putting an index by datavalue on this table? Once you've done that, then changing random_page_cost will make using that index look less expensive. Sorting chews through a good bit of CPU time, and that's where all of your runtime is being spent at--once you increase work_mem up very high that is. > I'm wondering now what changes might get this query to run in less than > one second. If all the data is in memory, then will faster CPU and > memory be the things that help? You're trying to fix a fundamental design issue with hardware. That usually doesn't go well. Once you get a box big enough to hold the whole database in RAM, beyond that the differences between server systems are relatively small. -- Greg Smith 2ndQuadrant US greg@2ndQuadrant.com Baltimore, MD PostgreSQL Training, Services, and 24x7 Support www.2ndQuadrant.com {noformat} and {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 8:57:32 PM PDT To: Greg Smith Cc: Mike McCann , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden , Jeff Janes On Sun, May 19, 2013 at 8:44 PM, Greg Smith wrote: >On 5/13/13 6:36 PM, Mike McCann wrote: >> >> stoqs_march2013_s=# explain analyze select * from >> stoqs_measuredparameter order by datavalue; >> >> QUERY PLAN >> >>------------------------------------------------------------------------------------------------------------------------------------------ >> Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual >> time=2503.078..2937.130 rows=3381814 loops=1) >> Sort Key: datavalue >> Sort Method: quicksort Memory: 362509kB >> -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 >> rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) >> Total runtime: 3094.601 ms >> (5 rows) >> >> I tried changing random_page_cost to from 4 to 1 and saw no change. > > > Have you tried putting an index by datavalue on this table? Once you've > done that, then changing random_page_cost will make using that index look > less expensive. Sorting chews through a good bit of CPU time, and that's > where all of your runtime is being spent at--once you increase work_mem up > very high that is. This++ plus cluster on that index if you can. {noformat} h4. 2 July 2013 We made the decision made to purchase a server with a FusionIO card. Server is now set up on the network with name "kraken". Here is the [confluence page for kraken's configuration settings| PostgreSQL Server kraken setup]. Setting up ability to run pgbench-tools on all the various servers. This may be done per user account by first installing from git and following the instructions in README.rst; however, the scripts need to run commands requiring createdb and vacuum privileges, so it's best to run as user postgres. So, logging into pgtest as user postgres and getting things set up: {noformat} -bash-4.1$ pwd /var/lib/pgsql -bash-4.1$ mkdir dev -bash-4.1$ cd dev -bash-4.1$ git clone git://git.postgresql.org/git/pgbench-tools.git Initialized empty Git repository in /var/lib/pgsql/dev/pgbench-tools/.git/ remote: Counting objects: 108, done. remote: Compressing objects: 100% (106/106), done. remote: Total 108 (delta 54), reused 0 (delta 0) Receiving objects: 100% (108/108), 24.61 KiB, done. Resolving deltas: 100% (54/54), done. -bash-4.1$ createdb results -bash-4.1$ createdb pgbench -bash-4.1$ cd pgbench-tools/ -bash-4.1$ psql -f init/resultdb.sql -d results psql:init/resultdb.sql:1: ERROR: table "timing" does not exist CREATE TABLE CREATE INDEX psql:init/resultdb.sql:12: ERROR: table "tests" does not exist psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_test_seq" for serial column "tests.test" psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_set_seq" for serial column "tests.set" CREATE TABLE psql:init/resultdb.sql:31: ERROR: table "testset" does not exist psql:init/resultdb.sql:36: NOTICE: CREATE TABLE will create implicit sequence "testset_set_seq" for serial column "testset.set" CREATE TABLE INSERT 0 1 psql:init/resultdb.sql:40: ERROR: table "test_bgwriter" does not exist CREATE TABLE -bash-4.1$ psql -c "UPDATE testset SET info='Initial test set' WHERE set=1" -d results UPDATE 1 {noformat} I think those ERROR messages can be ignored because the following tests work. The gnuplot package is needed for producing graphs of the results: {noformat} sudo -c "yum install gnuplot" {noformat} Edit the config file with hard path to pgbench and reduced number of scale and client runs so that we have a quick execution to just test things: {noformat} -bash-4.1$ mv config config.130702 -bash-4.1$ cp config.130702 config -bash-4.1$ vi config -bash-4.1$ diff config.130702 config 8c8 < PGBENCHBIN=`which pgbench` --- > PGBENCHBIN=/usr/pgsql-9.1/bin/pgbench 42,44c42,47 < SCALES="1 10 100 1000" < SETCLIENTS="1 2 4 8 16 32" < SETTIMES=3 --- > SCALES="1" > SETCLIENTS="1" > SETTIMES=1 > #SCALES="1 10 100 1000" > #SETCLIENTS="1 2 4 8 16 32" > #SETTIMES=3 {noformat} Edit scripts to use "local socket" connection for the postgres user: {noformat} -bash-4.1$ diff runset.130702 runset 19c19 < TESTPSQL="psql -h $TESTHOST -U $TESTUSER -p $TESTPORT -d $TESTDB" --- > TESTPSQL="psql -U $TESTUSER -p $TESTPORT -d $TESTDB" 24c24 < $PGBENCHBIN -i -s $SCALE -h $TESTHOST -U $TESTUSER -p $TESTPORT $TESTDB --- > $PGBENCHBIN -i -s $SCALE -U $TESTUSER -p $TESTPORT $TESTDB -bash-4.1$ diff benchwarmer.130702 benchwarmer 84,85c84,85 < TESTPSQL="psql -h $TESTHOST -U $TESTUSER -p $TESTPORT -d $TESTDB" < RESULTPSQL="psql -h $RESULTHOST -U $RESULTUSER -p $RESULTPORT -d $RESULTDB" --- > TESTPSQL="psql -U $TESTUSER -p $TESTPORT -d $TESTDB" > RESULTPSQL="psql -U $RESULTUSER -p $RESULTPORT -d $RESULTDB" 158c158 < $PGBENCHBIN -f $BASEDIR/$TESTDIR/$SCRIPT -s $SCALE -l -n $TESTLEN -U $TESTUSER -h $TESTHOST -p $TESTPORT -c $CLIENTS $WORKERS $TESTDB > results.txt & --- > $PGBENCHBIN -f $BASEDIR/$TESTDIR/$SCRIPT -s $SCALE -l -n $TESTLEN -U $TESTUSER -p $TESTPORT -c $CLIENTS $WORKERS $TESTDB > results.txt & {noformat} ]]> 21365311 21398036 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 h4. 16 May 2013 Another post to the pgsql-perform mail list asking for opinions on a more conventional server configuration: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 3:36:03 PM PDT To: pgsql-performance@postgresql.org Cc: Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On May 7, 2013, at 4:21 PM, Jeff Janes wrote: On Thu, May 2, 2013 at 6:35 PM, Scott Marlowe wrote: On Thu, May 2, 2013 at 5:11 PM, Mike McCann wrote: > Hello, > > We are in the fortunate situation of having more money than time to help > solve our PostgreSQL 9.1 performance problem. > > Our server hosts databases that are about 1 GB in size with the largest > tables having order 10 million 20-byte indexed records. The data are loaded > once and then read from a web app and other client programs. Some of the > queries execute ORDER BY on the results. There are typically less than a > dozen read-only concurrent connections to any one database. I wouldn't count on this being a problem that can be fixed merely by throwing money at it. How many rows does any one of these queries need to access and then ORDER BY? ... > > HP ProLiant DL360p Gen 8 > Dual Intel Xeon 2.4GHz 4-core E5-2609 CPUs > 64GB RAM > 2x146GB 15K SAS hard drives > 3x200GB SATA SLC SSDs > + the usual accessories (optical drive, rail kit, dual power supplies) If your DB is 1G, and will grow to 10G then the IO shouldn't be any problem, as the whole db should be cached in memory. But it can take a surprisingly long time to get it cached in the first place, from a cold start. If that is the problem, pg_prewarm could help. Cheers, Jeff Thank you everyone for your suggestions. It's clear that our current read performance was not limited by hardware. An 'explain analyze' for a sample query is: stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=541002.15..549456.68 rows=3381814 width=20) (actual time=6254.780..7244.074 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..354.385 rows=3381814 loops=1) Total runtime: 7425.854 ms (5 rows) Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? We have an alternate (a bit more conventional) server configuration that we are considering: HP ProLiant DL360p Gen 8 Dual Intel Xeon 3.3GHz 4-core E5-2643 CPUs 128GB PC3-12800 RAM 16x146GB 15K SAS hard drives HP Smart Array P822/2GB FBWC controller + P420i w/ 2GB FBWC + the usual accessories (optical drive, rail kit, dual power supplies) All suggestions welcomed! -Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} The responses: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 4:58:17 PM PDT To: Jeff Janes Cc: "pgsql-performance@postgresql.org" , Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden On May 13, 2013, at 4:24 PM, Jeff Janes wrote: > On Mon, May 13, 2013 at 3:36 PM, Mike McCann wrote: > > Increasing work_mem to 355 MB improves the performance by a factor of 2: > > stoqs_march2013_s=# set work_mem='355MB'; > SET > stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > > I tried changing random_page_cost to from 4 to 1 and saw no change. > > I'm wondering now what changes might get this query to run in less than one second. I think you are worrying about the wrong thing here. What is a web app going to do with 3,381,814 rows, once it obtains them? Your current testing is not testing the time it takes to stream that data to the client, or for the client to do something meaningful with that data. If you only plan to actually fetch a few dozen of those rows, then you probably need to incorporate that into your test, either by using a LIMIT, or by using a mock-up of the actual application to do some timings. Also, what is the type and collation of the column you are sorting on? non-'C' collations of text columns sort about 3 times slower than 'C' collation does. If all the data is in memory, then will faster CPU and memory be the things that help? Yes, those would help (it is not clear to me which of the two would help more), but I think you need to rethink your design of sending the entire database table to the application server for each page-view. Cheers, Jeff Hi Jeff, The datavalue column is double precision: stoqs_march2013_s=# \d+ stoqs_measuredparameter Table "public.stoqs_measuredparameter" Column | Type | Modifiers | Storage | Description ----------------+------------------+----------------------------------------------------------------------+---------+------------- id | integer | not null default nextval('stoqs_measuredparameter_id_seq'::regclass) | plain | measurement_id | integer | not null | plain | parameter_id | integer | not null | plain | datavalue | double precision | not null | plain | Indexes: "stoqs_measuredparameter_pkey" PRIMARY KEY, btree (id) "stoqs_measuredparameter_measurement_id_parameter_id_key" UNIQUE CONSTRAINT, btree (measurement_id, parameter_id) "stoqs_measuredparameter_datavalue" btree (datavalue) "stoqs_measuredparameter_measurement_id" btree (measurement_id) "stoqs_measuredparameter_parameter_id" btree (parameter_id) Foreign-key constraints: "stoqs_measuredparameter_measurement_id_fkey" FOREIGN KEY (measurement_id) REFERENCES stoqs_measurement(id) DEFERRABLE INITIALLY DEFERRED "stoqs_measuredparameter_parameter_id_fkey" FOREIGN KEY (parameter_id) REFERENCES stoqs_parameter(id) DEFERRABLE INITIALLY DEFERRED Has OIDs: no Thanks for the suggestion and advice to examine the web app performance. We've actually taken quite a few steps to optimize how the web app works. The example query I provided is a simple worst-case one that we can use to help us decide on the proper hardware. An actual query performed by the web app is: stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_march2013_s-# stoqs_parameter.name AS parameter__name, stoqs_march2013_s-# stoqs_parameter.standard_name AS parameter__standard_name, stoqs_march2013_s-# stoqs_measurement.depth AS measurement__depth, stoqs_march2013_s-# stoqs_measurement.geom AS measurement__geom, stoqs_march2013_s-# stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_march2013_s-# stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_march2013_s-# stoqs_measuredparameter.datavalue AS datavalue, stoqs_march2013_s-# stoqs_parameter.units AS parameter__units stoqs_march2013_s-# FROM stoqs_parameter p1, stoqs_march2013_s-# stoqs_measuredparameter stoqs_march2013_s-# INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) stoqs_march2013_s-# INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) stoqs_march2013_s-# INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) stoqs_march2013_s-# INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) stoqs_march2013_s-# INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) stoqs_march2013_s-# INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id stoqs_march2013_s-# WHERE (p1.name = 'sea_water_sigma_t') stoqs_march2013_s-# AND (mp1.datavalue > 25.19) stoqs_march2013_s-# AND (mp1.datavalue < 26.01) stoqs_march2013_s-# AND (mp1.parameter_id = p1.id) stoqs_march2013_s-# AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' stoqs_march2013_s(# AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' stoqs_march2013_s(# AND stoqs_parameter.name IN ('fl700_uncorr') stoqs_march2013_s(# AND stoqs_measurement.depth >= -1.88 stoqs_march2013_s(# AND stoqs_platform.name IN ('dorado') stoqs_march2013_s(# AND stoqs_measurement.depth <= 83.57) stoqs_march2013_s-# ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=770.211..770.211 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=770.200..770.200 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=770.036..770.036 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=91.369..680.072 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=91.346..577.633 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=91.280..531.408 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=91.202..116.140 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.002..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=91.194..109.846 rows=20170 loop s=1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=90.794..90.794 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=62.769..62.769 rows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=27.412..2 7.412 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.008..0. 017 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.012..0.012 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.007..0.010 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.004..0.004 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.004..0.004 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.151..0.151 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.035..0.145 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.005..0.069 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.014..0.014 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.008..0.012 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 770.445 ms (42 rows) We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Thanks, Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} Another response: {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 6:09:58 PM PDT To: Mike McCann Cc: Jeff Janes , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden On Mon, May 13, 2013 at 5:58 PM, Mike McCann wrote: > We assume that steps taken to improve the worst-case query scenario will > also improve these kind of queries. If anything above pops out as needing > better planning please let us know that too! Bad assumption. If your real workload will be queries like the one here that takes 700 ms, but you'll be running 10,000 of them a second, you're tuning / hardware choices are going to be much different then if your query is going to be the previous 7 second one. Use realistic queries, not ones that are nothing like what your real ones will be. then use pgbench and its ability to run custom sql scripts to get a REAL idea how your hardware performs. Note that if you will run the slow query you posted like once a minute and roll it up or cache it then don't get too worried about it. Pay attention to the queries that will add up, in aggregate, to your greatest load. {noformat} Not a whole lot of opinions except that tuning a server requires diligent work and performance is something one must do some work to achieve, and cannot be had by simply throwing money at the problem. h4. 20 May 2013 A few more responses over the weekend: {noformat} From: Greg Smith Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 7:44:16 PM PDT To: Mike McCann Cc: pgsql-performance@postgresql.org, Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On 5/13/13 6:36 PM, Mike McCann wrote: > stoqs_march2013_s=# explain analyze select * from > stoqs_measuredparameter order by datavalue; > > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual > time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 > rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > I tried changing random_page_cost to from 4 to 1 and saw no change. Have you tried putting an index by datavalue on this table? Once you've done that, then changing random_page_cost will make using that index look less expensive. Sorting chews through a good bit of CPU time, and that's where all of your runtime is being spent at--once you increase work_mem up very high that is. > I'm wondering now what changes might get this query to run in less than > one second. If all the data is in memory, then will faster CPU and > memory be the things that help? You're trying to fix a fundamental design issue with hardware. That usually doesn't go well. Once you get a box big enough to hold the whole database in RAM, beyond that the differences between server systems are relatively small. -- Greg Smith 2ndQuadrant US greg@2ndQuadrant.com Baltimore, MD PostgreSQL Training, Services, and 24x7 Support www.2ndQuadrant.com {noformat} and {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 8:57:32 PM PDT To: Greg Smith Cc: Mike McCann , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden , Jeff Janes On Sun, May 19, 2013 at 8:44 PM, Greg Smith wrote: >On 5/13/13 6:36 PM, Mike McCann wrote: >> >> stoqs_march2013_s=# explain analyze select * from >> stoqs_measuredparameter order by datavalue; >> >> QUERY PLAN >> >>------------------------------------------------------------------------------------------------------------------------------------------ >> Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual >> time=2503.078..2937.130 rows=3381814 loops=1) >> Sort Key: datavalue >> Sort Method: quicksort Memory: 362509kB >> -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 >> rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) >> Total runtime: 3094.601 ms >> (5 rows) >> >> I tried changing random_page_cost to from 4 to 1 and saw no change. > > > Have you tried putting an index by datavalue on this table? Once you've > done that, then changing random_page_cost will make using that index look > less expensive. Sorting chews through a good bit of CPU time, and that's > where all of your runtime is being spent at--once you increase work_mem up > very high that is. This++ plus cluster on that index if you can. {noformat} h4. 2 July 2013 We made the decision made to purchase a server with a FusionIO card. Server is now set up on the network with name "kraken". Here is the [confluence page for kraken's configuration settings| PostgreSQL Server kraken setup]. Setting up ability to run pgbench-tools on all the various servers. This may be done per user account by first installing from git and following the instructions in README.rst; however, the scripts need to run commands requiring createdb and vacuum privileges, so it's best to run as user postgres. So, logging into pgtest as user postgres and getting things set up: {noformat} -bash-4.1$ pwd /var/lib/pgsql -bash-4.1$ mkdir dev -bash-4.1$ cd dev -bash-4.1$ git clone git://git.postgresql.org/git/pgbench-tools.git Initialized empty Git repository in /var/lib/pgsql/dev/pgbench-tools/.git/ remote: Counting objects: 108, done. remote: Compressing objects: 100% (106/106), done. remote: Total 108 (delta 54), reused 0 (delta 0) Receiving objects: 100% (108/108), 24.61 KiB, done. Resolving deltas: 100% (54/54), done. -bash-4.1$ createdb results -bash-4.1$ createdb pgbench -bash-4.1$ cd pgbench-tools/ -bash-4.1$ psql -f init/resultdb.sql -d results psql:init/resultdb.sql:1: ERROR: table "timing" does not exist CREATE TABLE CREATE INDEX psql:init/resultdb.sql:12: ERROR: table "tests" does not exist psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_test_seq" for serial column "tests.test" psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_set_seq" for serial column "tests.set" CREATE TABLE psql:init/resultdb.sql:31: ERROR: table "testset" does not exist psql:init/resultdb.sql:36: NOTICE: CREATE TABLE will create implicit sequence "testset_set_seq" for serial column "testset.set" CREATE TABLE INSERT 0 1 psql:init/resultdb.sql:40: ERROR: table "test_bgwriter" does not exist CREATE TABLE -bash-4.1$ psql -c "UPDATE testset SET info='Initial test set' WHERE set=1" -d results UPDATE 1 {noformat} I think those ERROR messages can be ignored because the following tests work. The gnuplot package is needed for producing graphs of the results: {noformat} sudo -c "yum install gnuplot" {noformat} Edit the config file to reduce the number of scale and client runs so that we have a quick execution: {noformat} <114 pgtest.shore.mbari.org /dev/pgbench-tools> mv config config.130702 <115 pgtest.shore.mbari.org /dev/pgbench-tools> cp config.130702 config <116 pgtest.shore.mbari.org /dev/pgbench-tools> vi config <117 pgtest.shore.mbari.org /dev/pgbench-tools> diff config config.130702 42,45c42,43 < SCALES="1 10" < #SCALES="1 10 100 1000" < SETCLIENTS="1 2" < #SETCLIENTS="1 2 4 8 16 32" --- > SCALES="1 10 100 1000" > SETCLIENTS="1 2 4 8 16 32" {noformat}]]> 21365309 9404550 9371795 9404549 9371794 21398026 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 h4. 16 May 2013 Another post to the pgsql-perform mail list asking for opinions on a more conventional server configuration: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 3:36:03 PM PDT To: pgsql-performance@postgresql.org Cc: Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On May 7, 2013, at 4:21 PM, Jeff Janes wrote: On Thu, May 2, 2013 at 6:35 PM, Scott Marlowe wrote: On Thu, May 2, 2013 at 5:11 PM, Mike McCann wrote: > Hello, > > We are in the fortunate situation of having more money than time to help > solve our PostgreSQL 9.1 performance problem. > > Our server hosts databases that are about 1 GB in size with the largest > tables having order 10 million 20-byte indexed records. The data are loaded > once and then read from a web app and other client programs. Some of the > queries execute ORDER BY on the results. There are typically less than a > dozen read-only concurrent connections to any one database. I wouldn't count on this being a problem that can be fixed merely by throwing money at it. How many rows does any one of these queries need to access and then ORDER BY? ... > > HP ProLiant DL360p Gen 8 > Dual Intel Xeon 2.4GHz 4-core E5-2609 CPUs > 64GB RAM > 2x146GB 15K SAS hard drives > 3x200GB SATA SLC SSDs > + the usual accessories (optical drive, rail kit, dual power supplies) If your DB is 1G, and will grow to 10G then the IO shouldn't be any problem, as the whole db should be cached in memory. But it can take a surprisingly long time to get it cached in the first place, from a cold start. If that is the problem, pg_prewarm could help. Cheers, Jeff Thank you everyone for your suggestions. It's clear that our current read performance was not limited by hardware. An 'explain analyze' for a sample query is: stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=541002.15..549456.68 rows=3381814 width=20) (actual time=6254.780..7244.074 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..354.385 rows=3381814 loops=1) Total runtime: 7425.854 ms (5 rows) Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? We have an alternate (a bit more conventional) server configuration that we are considering: HP ProLiant DL360p Gen 8 Dual Intel Xeon 3.3GHz 4-core E5-2643 CPUs 128GB PC3-12800 RAM 16x146GB 15K SAS hard drives HP Smart Array P822/2GB FBWC controller + P420i w/ 2GB FBWC + the usual accessories (optical drive, rail kit, dual power supplies) All suggestions welcomed! -Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} The responses: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 4:58:17 PM PDT To: Jeff Janes Cc: "pgsql-performance@postgresql.org" , Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden On May 13, 2013, at 4:24 PM, Jeff Janes wrote: > On Mon, May 13, 2013 at 3:36 PM, Mike McCann wrote: > > Increasing work_mem to 355 MB improves the performance by a factor of 2: > > stoqs_march2013_s=# set work_mem='355MB'; > SET > stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > > I tried changing random_page_cost to from 4 to 1 and saw no change. > > I'm wondering now what changes might get this query to run in less than one second. I think you are worrying about the wrong thing here. What is a web app going to do with 3,381,814 rows, once it obtains them? Your current testing is not testing the time it takes to stream that data to the client, or for the client to do something meaningful with that data. If you only plan to actually fetch a few dozen of those rows, then you probably need to incorporate that into your test, either by using a LIMIT, or by using a mock-up of the actual application to do some timings. Also, what is the type and collation of the column you are sorting on? non-'C' collations of text columns sort about 3 times slower than 'C' collation does. If all the data is in memory, then will faster CPU and memory be the things that help? Yes, those would help (it is not clear to me which of the two would help more), but I think you need to rethink your design of sending the entire database table to the application server for each page-view. Cheers, Jeff Hi Jeff, The datavalue column is double precision: stoqs_march2013_s=# \d+ stoqs_measuredparameter Table "public.stoqs_measuredparameter" Column | Type | Modifiers | Storage | Description ----------------+------------------+----------------------------------------------------------------------+---------+------------- id | integer | not null default nextval('stoqs_measuredparameter_id_seq'::regclass) | plain | measurement_id | integer | not null | plain | parameter_id | integer | not null | plain | datavalue | double precision | not null | plain | Indexes: "stoqs_measuredparameter_pkey" PRIMARY KEY, btree (id) "stoqs_measuredparameter_measurement_id_parameter_id_key" UNIQUE CONSTRAINT, btree (measurement_id, parameter_id) "stoqs_measuredparameter_datavalue" btree (datavalue) "stoqs_measuredparameter_measurement_id" btree (measurement_id) "stoqs_measuredparameter_parameter_id" btree (parameter_id) Foreign-key constraints: "stoqs_measuredparameter_measurement_id_fkey" FOREIGN KEY (measurement_id) REFERENCES stoqs_measurement(id) DEFERRABLE INITIALLY DEFERRED "stoqs_measuredparameter_parameter_id_fkey" FOREIGN KEY (parameter_id) REFERENCES stoqs_parameter(id) DEFERRABLE INITIALLY DEFERRED Has OIDs: no Thanks for the suggestion and advice to examine the web app performance. We've actually taken quite a few steps to optimize how the web app works. The example query I provided is a simple worst-case one that we can use to help us decide on the proper hardware. An actual query performed by the web app is: stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_march2013_s-# stoqs_parameter.name AS parameter__name, stoqs_march2013_s-# stoqs_parameter.standard_name AS parameter__standard_name, stoqs_march2013_s-# stoqs_measurement.depth AS measurement__depth, stoqs_march2013_s-# stoqs_measurement.geom AS measurement__geom, stoqs_march2013_s-# stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_march2013_s-# stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_march2013_s-# stoqs_measuredparameter.datavalue AS datavalue, stoqs_march2013_s-# stoqs_parameter.units AS parameter__units stoqs_march2013_s-# FROM stoqs_parameter p1, stoqs_march2013_s-# stoqs_measuredparameter stoqs_march2013_s-# INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) stoqs_march2013_s-# INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) stoqs_march2013_s-# INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) stoqs_march2013_s-# INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) stoqs_march2013_s-# INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) stoqs_march2013_s-# INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id stoqs_march2013_s-# WHERE (p1.name = 'sea_water_sigma_t') stoqs_march2013_s-# AND (mp1.datavalue > 25.19) stoqs_march2013_s-# AND (mp1.datavalue < 26.01) stoqs_march2013_s-# AND (mp1.parameter_id = p1.id) stoqs_march2013_s-# AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' stoqs_march2013_s(# AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' stoqs_march2013_s(# AND stoqs_parameter.name IN ('fl700_uncorr') stoqs_march2013_s(# AND stoqs_measurement.depth >= -1.88 stoqs_march2013_s(# AND stoqs_platform.name IN ('dorado') stoqs_march2013_s(# AND stoqs_measurement.depth <= 83.57) stoqs_march2013_s-# ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=770.211..770.211 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=770.200..770.200 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=770.036..770.036 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=91.369..680.072 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=91.346..577.633 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=91.280..531.408 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=91.202..116.140 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.002..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=91.194..109.846 rows=20170 loop s=1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=90.794..90.794 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=62.769..62.769 rows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=27.412..2 7.412 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.008..0. 017 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.012..0.012 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.007..0.010 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.004..0.004 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.004..0.004 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.151..0.151 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.035..0.145 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.005..0.069 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.014..0.014 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.008..0.012 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 770.445 ms (42 rows) We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Thanks, Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} Another response: {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 6:09:58 PM PDT To: Mike McCann Cc: Jeff Janes , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden On Mon, May 13, 2013 at 5:58 PM, Mike McCann wrote: > We assume that steps taken to improve the worst-case query scenario will > also improve these kind of queries. If anything above pops out as needing > better planning please let us know that too! Bad assumption. If your real workload will be queries like the one here that takes 700 ms, but you'll be running 10,000 of them a second, you're tuning / hardware choices are going to be much different then if your query is going to be the previous 7 second one. Use realistic queries, not ones that are nothing like what your real ones will be. then use pgbench and its ability to run custom sql scripts to get a REAL idea how your hardware performs. Note that if you will run the slow query you posted like once a minute and roll it up or cache it then don't get too worried about it. Pay attention to the queries that will add up, in aggregate, to your greatest load. {noformat} Not a whole lot of opinions except that tuning a server requires diligent work and performance is something one must do some work to achieve, and cannot be had by simply throwing money at the problem. h4. 20 May 2013 A few more responses over the weekend: {noformat} From: Greg Smith Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 7:44:16 PM PDT To: Mike McCann Cc: pgsql-performance@postgresql.org, Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On 5/13/13 6:36 PM, Mike McCann wrote: > stoqs_march2013_s=# explain analyze select * from > stoqs_measuredparameter order by datavalue; > > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual > time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 > rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > I tried changing random_page_cost to from 4 to 1 and saw no change. Have you tried putting an index by datavalue on this table? Once you've done that, then changing random_page_cost will make using that index look less expensive. Sorting chews through a good bit of CPU time, and that's where all of your runtime is being spent at--once you increase work_mem up very high that is. > I'm wondering now what changes might get this query to run in less than > one second. If all the data is in memory, then will faster CPU and > memory be the things that help? You're trying to fix a fundamental design issue with hardware. That usually doesn't go well. Once you get a box big enough to hold the whole database in RAM, beyond that the differences between server systems are relatively small. -- Greg Smith 2ndQuadrant US greg@2ndQuadrant.com Baltimore, MD PostgreSQL Training, Services, and 24x7 Support www.2ndQuadrant.com {noformat} and {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 8:57:32 PM PDT To: Greg Smith Cc: Mike McCann , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden , Jeff Janes On Sun, May 19, 2013 at 8:44 PM, Greg Smith wrote: >On 5/13/13 6:36 PM, Mike McCann wrote: >> >> stoqs_march2013_s=# explain analyze select * from >> stoqs_measuredparameter order by datavalue; >> >> QUERY PLAN >> >>------------------------------------------------------------------------------------------------------------------------------------------ >> Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual >> time=2503.078..2937.130 rows=3381814 loops=1) >> Sort Key: datavalue >> Sort Method: quicksort Memory: 362509kB >> -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 >> rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) >> Total runtime: 3094.601 ms >> (5 rows) >> >> I tried changing random_page_cost to from 4 to 1 and saw no change. > > > Have you tried putting an index by datavalue on this table? Once you've > done that, then changing random_page_cost will make using that index look > less expensive. Sorting chews through a good bit of CPU time, and that's > where all of your runtime is being spent at--once you increase work_mem up > very high that is. This++ plus cluster on that index if you can. {noformat} h4. 2 July 2013 Decision made to purchase server with FusionIO card. Server is now set up on the network with name "kraken". Here is the [confluence page for kraken's configuration settings| PostgreSQL Server kraken setup]. Setting up ability to run pgbench-tools on all the various servers. This can be done per user account by first installing from git and following the instructions in README.rst: {noformat} <108 pgtest.shore.mbari.org /mccann/dev> git clone git://git.postgresql.org/git/pgbench-tools.git Initialized empty Git repository in /u/mccann/dev/pgbench-tools/.git/ remote: Counting objects: 108, done. remote: Compressing objects: 100% (106/106), done. remote: Total 108 (delta 54), reused 0 (delta 0) Receiving objects: 100% (108/108), 24.61 KiB, done. Resolving deltas: 100% (54/54), done. {noformat} The postgresql server on pgtest has higher security than the postgresql servers on our development VMs so the pgbench and results databases must be created as user postgres and then ownership granted to a shared account (stoqsadm) that we can use for running tests. Here's how the databases were created and set up: {noformat} su postgres psql # create database pgbench owner=stoqsadm; CREATE DATABASE # create database results owner=stoqsadm; CREATE DATABASE # \c pgbench You are now connected to database "pgbench" as user "postgres". pgbench=# grant all on all tables in schema public to stoqsadm; GRANT pgbench=# \c results You are now connected to database "results" as user "postgres". results=# grant all on all tables in schema public to stoqsadm; GRANT {noformat} Now we can run commands against these databases using the pgbench-tools commands: {noformat} <112 pgtest.shore.mbari.org /dev/pgbench-tools> psql -f init/resultdb.sql -d results -h localhost -U stoqsadm Password for user stoqsadm: psql:init/resultdb.sql:1: ERROR: table "timing" does not exist CREATE TABLE CREATE INDEX psql:init/resultdb.sql:12: ERROR: table "tests" does not exist psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_test_seq" for serial column "tests.test" psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_set_seq" for serial column "tests.set" CREATE TABLE psql:init/resultdb.sql:31: ERROR: table "testset" does not exist psql:init/resultdb.sql:36: NOTICE: CREATE TABLE will create implicit sequence "testset_set_seq" for serial column "testset.set" CREATE TABLE INSERT 0 1 psql:init/resultdb.sql:40: ERROR: table "test_bgwriter" does not exist CREATE TABLE {noformat} The additional options "-h localhost -U stoqsadm" must be added to the example commands given in the pgbench-tools documentation. ]]> 21365299 9404568 9371818 9404569 9371819 21398024 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 h4. 16 May 2013 Another post to the pgsql-perform mail list asking for opinions on a more conventional server configuration: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 3:36:03 PM PDT To: pgsql-performance@postgresql.org Cc: Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On May 7, 2013, at 4:21 PM, Jeff Janes wrote: On Thu, May 2, 2013 at 6:35 PM, Scott Marlowe wrote: On Thu, May 2, 2013 at 5:11 PM, Mike McCann wrote: > Hello, > > We are in the fortunate situation of having more money than time to help > solve our PostgreSQL 9.1 performance problem. > > Our server hosts databases that are about 1 GB in size with the largest > tables having order 10 million 20-byte indexed records. The data are loaded > once and then read from a web app and other client programs. Some of the > queries execute ORDER BY on the results. There are typically less than a > dozen read-only concurrent connections to any one database. I wouldn't count on this being a problem that can be fixed merely by throwing money at it. How many rows does any one of these queries need to access and then ORDER BY? ... > > HP ProLiant DL360p Gen 8 > Dual Intel Xeon 2.4GHz 4-core E5-2609 CPUs > 64GB RAM > 2x146GB 15K SAS hard drives > 3x200GB SATA SLC SSDs > + the usual accessories (optical drive, rail kit, dual power supplies) If your DB is 1G, and will grow to 10G then the IO shouldn't be any problem, as the whole db should be cached in memory. But it can take a surprisingly long time to get it cached in the first place, from a cold start. If that is the problem, pg_prewarm could help. Cheers, Jeff Thank you everyone for your suggestions. It's clear that our current read performance was not limited by hardware. An 'explain analyze' for a sample query is: stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=541002.15..549456.68 rows=3381814 width=20) (actual time=6254.780..7244.074 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..354.385 rows=3381814 loops=1) Total runtime: 7425.854 ms (5 rows) Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? We have an alternate (a bit more conventional) server configuration that we are considering: HP ProLiant DL360p Gen 8 Dual Intel Xeon 3.3GHz 4-core E5-2643 CPUs 128GB PC3-12800 RAM 16x146GB 15K SAS hard drives HP Smart Array P822/2GB FBWC controller + P420i w/ 2GB FBWC + the usual accessories (optical drive, rail kit, dual power supplies) All suggestions welcomed! -Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} The responses: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 4:58:17 PM PDT To: Jeff Janes Cc: "pgsql-performance@postgresql.org" , Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden On May 13, 2013, at 4:24 PM, Jeff Janes wrote: > On Mon, May 13, 2013 at 3:36 PM, Mike McCann wrote: > > Increasing work_mem to 355 MB improves the performance by a factor of 2: > > stoqs_march2013_s=# set work_mem='355MB'; > SET > stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > > I tried changing random_page_cost to from 4 to 1 and saw no change. > > I'm wondering now what changes might get this query to run in less than one second. I think you are worrying about the wrong thing here. What is a web app going to do with 3,381,814 rows, once it obtains them? Your current testing is not testing the time it takes to stream that data to the client, or for the client to do something meaningful with that data. If you only plan to actually fetch a few dozen of those rows, then you probably need to incorporate that into your test, either by using a LIMIT, or by using a mock-up of the actual application to do some timings. Also, what is the type and collation of the column you are sorting on? non-'C' collations of text columns sort about 3 times slower than 'C' collation does. If all the data is in memory, then will faster CPU and memory be the things that help? Yes, those would help (it is not clear to me which of the two would help more), but I think you need to rethink your design of sending the entire database table to the application server for each page-view. Cheers, Jeff Hi Jeff, The datavalue column is double precision: stoqs_march2013_s=# \d+ stoqs_measuredparameter Table "public.stoqs_measuredparameter" Column | Type | Modifiers | Storage | Description ----------------+------------------+----------------------------------------------------------------------+---------+------------- id | integer | not null default nextval('stoqs_measuredparameter_id_seq'::regclass) | plain | measurement_id | integer | not null | plain | parameter_id | integer | not null | plain | datavalue | double precision | not null | plain | Indexes: "stoqs_measuredparameter_pkey" PRIMARY KEY, btree (id) "stoqs_measuredparameter_measurement_id_parameter_id_key" UNIQUE CONSTRAINT, btree (measurement_id, parameter_id) "stoqs_measuredparameter_datavalue" btree (datavalue) "stoqs_measuredparameter_measurement_id" btree (measurement_id) "stoqs_measuredparameter_parameter_id" btree (parameter_id) Foreign-key constraints: "stoqs_measuredparameter_measurement_id_fkey" FOREIGN KEY (measurement_id) REFERENCES stoqs_measurement(id) DEFERRABLE INITIALLY DEFERRED "stoqs_measuredparameter_parameter_id_fkey" FOREIGN KEY (parameter_id) REFERENCES stoqs_parameter(id) DEFERRABLE INITIALLY DEFERRED Has OIDs: no Thanks for the suggestion and advice to examine the web app performance. We've actually taken quite a few steps to optimize how the web app works. The example query I provided is a simple worst-case one that we can use to help us decide on the proper hardware. An actual query performed by the web app is: stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_march2013_s-# stoqs_parameter.name AS parameter__name, stoqs_march2013_s-# stoqs_parameter.standard_name AS parameter__standard_name, stoqs_march2013_s-# stoqs_measurement.depth AS measurement__depth, stoqs_march2013_s-# stoqs_measurement.geom AS measurement__geom, stoqs_march2013_s-# stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_march2013_s-# stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_march2013_s-# stoqs_measuredparameter.datavalue AS datavalue, stoqs_march2013_s-# stoqs_parameter.units AS parameter__units stoqs_march2013_s-# FROM stoqs_parameter p1, stoqs_march2013_s-# stoqs_measuredparameter stoqs_march2013_s-# INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) stoqs_march2013_s-# INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) stoqs_march2013_s-# INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) stoqs_march2013_s-# INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) stoqs_march2013_s-# INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) stoqs_march2013_s-# INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id stoqs_march2013_s-# WHERE (p1.name = 'sea_water_sigma_t') stoqs_march2013_s-# AND (mp1.datavalue > 25.19) stoqs_march2013_s-# AND (mp1.datavalue < 26.01) stoqs_march2013_s-# AND (mp1.parameter_id = p1.id) stoqs_march2013_s-# AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' stoqs_march2013_s(# AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' stoqs_march2013_s(# AND stoqs_parameter.name IN ('fl700_uncorr') stoqs_march2013_s(# AND stoqs_measurement.depth >= -1.88 stoqs_march2013_s(# AND stoqs_platform.name IN ('dorado') stoqs_march2013_s(# AND stoqs_measurement.depth <= 83.57) stoqs_march2013_s-# ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=770.211..770.211 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=770.200..770.200 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=770.036..770.036 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=91.369..680.072 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=91.346..577.633 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=91.280..531.408 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=91.202..116.140 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.002..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=91.194..109.846 rows=20170 loop s=1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=90.794..90.794 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=62.769..62.769 rows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=27.412..2 7.412 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.008..0. 017 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.012..0.012 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.007..0.010 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.004..0.004 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.004..0.004 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.151..0.151 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.035..0.145 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.005..0.069 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.014..0.014 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.008..0.012 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 770.445 ms (42 rows) We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Thanks, Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} Another response: {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 6:09:58 PM PDT To: Mike McCann Cc: Jeff Janes , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden On Mon, May 13, 2013 at 5:58 PM, Mike McCann wrote: > We assume that steps taken to improve the worst-case query scenario will > also improve these kind of queries. If anything above pops out as needing > better planning please let us know that too! Bad assumption. If your real workload will be queries like the one here that takes 700 ms, but you'll be running 10,000 of them a second, you're tuning / hardware choices are going to be much different then if your query is going to be the previous 7 second one. Use realistic queries, not ones that are nothing like what your real ones will be. then use pgbench and its ability to run custom sql scripts to get a REAL idea how your hardware performs. Note that if you will run the slow query you posted like once a minute and roll it up or cache it then don't get too worried about it. Pay attention to the queries that will add up, in aggregate, to your greatest load. {noformat} Not a whole lot of opinions except that tuning a server requires diligent work and performance is something one must do some work to achieve, and cannot be had by simply throwing money at the problem. h4. 20 May 2013 A few more responses over the weekend: {noformat} From: Greg Smith Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 7:44:16 PM PDT To: Mike McCann Cc: pgsql-performance@postgresql.org, Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On 5/13/13 6:36 PM, Mike McCann wrote: > stoqs_march2013_s=# explain analyze select * from > stoqs_measuredparameter order by datavalue; > > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual > time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 > rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > I tried changing random_page_cost to from 4 to 1 and saw no change. Have you tried putting an index by datavalue on this table? Once you've done that, then changing random_page_cost will make using that index look less expensive. Sorting chews through a good bit of CPU time, and that's where all of your runtime is being spent at--once you increase work_mem up very high that is. > I'm wondering now what changes might get this query to run in less than > one second. If all the data is in memory, then will faster CPU and > memory be the things that help? You're trying to fix a fundamental design issue with hardware. That usually doesn't go well. Once you get a box big enough to hold the whole database in RAM, beyond that the differences between server systems are relatively small. -- Greg Smith 2ndQuadrant US greg@2ndQuadrant.com Baltimore, MD PostgreSQL Training, Services, and 24x7 Support www.2ndQuadrant.com {noformat} and {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 8:57:32 PM PDT To: Greg Smith Cc: Mike McCann , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden , Jeff Janes On Sun, May 19, 2013 at 8:44 PM, Greg Smith wrote: >On 5/13/13 6:36 PM, Mike McCann wrote: >> >> stoqs_march2013_s=# explain analyze select * from >> stoqs_measuredparameter order by datavalue; >> >> QUERY PLAN >> >>------------------------------------------------------------------------------------------------------------------------------------------ >> Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual >> time=2503.078..2937.130 rows=3381814 loops=1) >> Sort Key: datavalue >> Sort Method: quicksort Memory: 362509kB >> -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 >> rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) >> Total runtime: 3094.601 ms >> (5 rows) >> >> I tried changing random_page_cost to from 4 to 1 and saw no change. > > > Have you tried putting an index by datavalue on this table? Once you've > done that, then changing random_page_cost will make using that index look > less expensive. Sorting chews through a good bit of CPU time, and that's > where all of your runtime is being spent at--once you increase work_mem up > very high that is. This++ plus cluster on that index if you can. {noformat} h4. 2 July 2013 Decision made to purchase server with FusionIO card. Server is now set up on the network with name "kraken". Here is the [confluence page for kraken's configuration settings| PostgreSQL Server kraken setup]. Setting up ability to run pgbench-tools on all the various servers. This can be done per user account by first installing from git and following the instructions in README.rst: {noformat} <108 pgtest.shore.mbari.org /mccann/dev> git clone git://git.postgresql.org/git/pgbench-tools.git Initialized empty Git repository in /u/mccann/dev/pgbench-tools/.git/ remote: Counting objects: 108, done. remote: Compressing objects: 100% (106/106), done. remote: Total 108 (delta 54), reused 0 (delta 0) Receiving objects: 100% (108/108), 24.61 KiB, done. Resolving deltas: 100% (54/54), done. {noformat} The postgresql server on pgtest has higher security than the postgresql servers on our development VMs so the pgbench and results databases must be created as user postgres and then ownership granted to a shared account (stoqsadm) that we can use for running tests. Here's how the databases were created and set up: {noformat} su postgres psql # create database pgbench owner=stoqsadm; CREATE DATABASE # create database results owner=stoqsadm; CREATE DATABASE # \c pgbench You are now connected to database "pgbench" as user "postgres". pgbench=# grant all on all tables in schema public to stoqsadm; GRANT pgbench=# \c results You are now connected to database "results" as user "postgres". results=# grant all on all tables in schema public to stoqsadm; GRANT {noformat} Now we can run commands against these databases using the pgbench-tools commands: {noformat} <112 pgtest.shore.mbari.org /dev/pgbench-tools> psql -f init/resultdb.sql -d results -h localhost -U stoqsadm Password for user stoqsadm: psql:init/resultdb.sql:1: ERROR: table "timing" does not exist CREATE TABLE CREATE INDEX psql:init/resultdb.sql:12: ERROR: table "tests" does not exist psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_test_seq" for serial column "tests.test" psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_set_seq" for serial column "tests.set" CREATE TABLE psql:init/resultdb.sql:31: ERROR: table "testset" does not exist psql:init/resultdb.sql:36: NOTICE: CREATE TABLE will create implicit sequence "testset_set_seq" for serial column "testset.set" CREATE TABLE INSERT 0 1 psql:init/resultdb.sql:40: ERROR: table "test_bgwriter" does not exist CREATE TABLE {noformat} The additional options "-h localhost -U stoqsadm" must be added to the example commands given in the pgbench-tools documentation. ]]> 21365297 9404570 9371820 21398022 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 h4. 16 May 2013 Another post to the pgsql-perform mail list asking for opinions on a more conventional server configuration: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 3:36:03 PM PDT To: pgsql-performance@postgresql.org Cc: Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On May 7, 2013, at 4:21 PM, Jeff Janes wrote: On Thu, May 2, 2013 at 6:35 PM, Scott Marlowe wrote: On Thu, May 2, 2013 at 5:11 PM, Mike McCann wrote: > Hello, > > We are in the fortunate situation of having more money than time to help > solve our PostgreSQL 9.1 performance problem. > > Our server hosts databases that are about 1 GB in size with the largest > tables having order 10 million 20-byte indexed records. The data are loaded > once and then read from a web app and other client programs. Some of the > queries execute ORDER BY on the results. There are typically less than a > dozen read-only concurrent connections to any one database. I wouldn't count on this being a problem that can be fixed merely by throwing money at it. How many rows does any one of these queries need to access and then ORDER BY? ... > > HP ProLiant DL360p Gen 8 > Dual Intel Xeon 2.4GHz 4-core E5-2609 CPUs > 64GB RAM > 2x146GB 15K SAS hard drives > 3x200GB SATA SLC SSDs > + the usual accessories (optical drive, rail kit, dual power supplies) If your DB is 1G, and will grow to 10G then the IO shouldn't be any problem, as the whole db should be cached in memory. But it can take a surprisingly long time to get it cached in the first place, from a cold start. If that is the problem, pg_prewarm could help. Cheers, Jeff Thank you everyone for your suggestions. It's clear that our current read performance was not limited by hardware. An 'explain analyze' for a sample query is: stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=541002.15..549456.68 rows=3381814 width=20) (actual time=6254.780..7244.074 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..354.385 rows=3381814 loops=1) Total runtime: 7425.854 ms (5 rows) Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? We have an alternate (a bit more conventional) server configuration that we are considering: HP ProLiant DL360p Gen 8 Dual Intel Xeon 3.3GHz 4-core E5-2643 CPUs 128GB PC3-12800 RAM 16x146GB 15K SAS hard drives HP Smart Array P822/2GB FBWC controller + P420i w/ 2GB FBWC + the usual accessories (optical drive, rail kit, dual power supplies) All suggestions welcomed! -Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} The responses: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 4:58:17 PM PDT To: Jeff Janes Cc: "pgsql-performance@postgresql.org" , Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden On May 13, 2013, at 4:24 PM, Jeff Janes wrote: > On Mon, May 13, 2013 at 3:36 PM, Mike McCann wrote: > > Increasing work_mem to 355 MB improves the performance by a factor of 2: > > stoqs_march2013_s=# set work_mem='355MB'; > SET > stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > > I tried changing random_page_cost to from 4 to 1 and saw no change. > > I'm wondering now what changes might get this query to run in less than one second. I think you are worrying about the wrong thing here. What is a web app going to do with 3,381,814 rows, once it obtains them? Your current testing is not testing the time it takes to stream that data to the client, or for the client to do something meaningful with that data. If you only plan to actually fetch a few dozen of those rows, then you probably need to incorporate that into your test, either by using a LIMIT, or by using a mock-up of the actual application to do some timings. Also, what is the type and collation of the column you are sorting on? non-'C' collations of text columns sort about 3 times slower than 'C' collation does. If all the data is in memory, then will faster CPU and memory be the things that help? Yes, those would help (it is not clear to me which of the two would help more), but I think you need to rethink your design of sending the entire database table to the application server for each page-view. Cheers, Jeff Hi Jeff, The datavalue column is double precision: stoqs_march2013_s=# \d+ stoqs_measuredparameter Table "public.stoqs_measuredparameter" Column | Type | Modifiers | Storage | Description ----------------+------------------+----------------------------------------------------------------------+---------+------------- id | integer | not null default nextval('stoqs_measuredparameter_id_seq'::regclass) | plain | measurement_id | integer | not null | plain | parameter_id | integer | not null | plain | datavalue | double precision | not null | plain | Indexes: "stoqs_measuredparameter_pkey" PRIMARY KEY, btree (id) "stoqs_measuredparameter_measurement_id_parameter_id_key" UNIQUE CONSTRAINT, btree (measurement_id, parameter_id) "stoqs_measuredparameter_datavalue" btree (datavalue) "stoqs_measuredparameter_measurement_id" btree (measurement_id) "stoqs_measuredparameter_parameter_id" btree (parameter_id) Foreign-key constraints: "stoqs_measuredparameter_measurement_id_fkey" FOREIGN KEY (measurement_id) REFERENCES stoqs_measurement(id) DEFERRABLE INITIALLY DEFERRED "stoqs_measuredparameter_parameter_id_fkey" FOREIGN KEY (parameter_id) REFERENCES stoqs_parameter(id) DEFERRABLE INITIALLY DEFERRED Has OIDs: no Thanks for the suggestion and advice to examine the web app performance. We've actually taken quite a few steps to optimize how the web app works. The example query I provided is a simple worst-case one that we can use to help us decide on the proper hardware. An actual query performed by the web app is: stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_march2013_s-# stoqs_parameter.name AS parameter__name, stoqs_march2013_s-# stoqs_parameter.standard_name AS parameter__standard_name, stoqs_march2013_s-# stoqs_measurement.depth AS measurement__depth, stoqs_march2013_s-# stoqs_measurement.geom AS measurement__geom, stoqs_march2013_s-# stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_march2013_s-# stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_march2013_s-# stoqs_measuredparameter.datavalue AS datavalue, stoqs_march2013_s-# stoqs_parameter.units AS parameter__units stoqs_march2013_s-# FROM stoqs_parameter p1, stoqs_march2013_s-# stoqs_measuredparameter stoqs_march2013_s-# INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) stoqs_march2013_s-# INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) stoqs_march2013_s-# INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) stoqs_march2013_s-# INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) stoqs_march2013_s-# INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) stoqs_march2013_s-# INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id stoqs_march2013_s-# WHERE (p1.name = 'sea_water_sigma_t') stoqs_march2013_s-# AND (mp1.datavalue > 25.19) stoqs_march2013_s-# AND (mp1.datavalue < 26.01) stoqs_march2013_s-# AND (mp1.parameter_id = p1.id) stoqs_march2013_s-# AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' stoqs_march2013_s(# AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' stoqs_march2013_s(# AND stoqs_parameter.name IN ('fl700_uncorr') stoqs_march2013_s(# AND stoqs_measurement.depth >= -1.88 stoqs_march2013_s(# AND stoqs_platform.name IN ('dorado') stoqs_march2013_s(# AND stoqs_measurement.depth <= 83.57) stoqs_march2013_s-# ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=770.211..770.211 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=770.200..770.200 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=770.036..770.036 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=91.369..680.072 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=91.346..577.633 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=91.280..531.408 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=91.202..116.140 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.002..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=91.194..109.846 rows=20170 loop s=1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=90.794..90.794 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=62.769..62.769 rows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=27.412..2 7.412 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.008..0. 017 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.012..0.012 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.007..0.010 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.004..0.004 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.004..0.004 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.151..0.151 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.035..0.145 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.005..0.069 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.014..0.014 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.008..0.012 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 770.445 ms (42 rows) We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Thanks, Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} Another response: {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 6:09:58 PM PDT To: Mike McCann Cc: Jeff Janes , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden On Mon, May 13, 2013 at 5:58 PM, Mike McCann wrote: > We assume that steps taken to improve the worst-case query scenario will > also improve these kind of queries. If anything above pops out as needing > better planning please let us know that too! Bad assumption. If your real workload will be queries like the one here that takes 700 ms, but you'll be running 10,000 of them a second, you're tuning / hardware choices are going to be much different then if your query is going to be the previous 7 second one. Use realistic queries, not ones that are nothing like what your real ones will be. then use pgbench and its ability to run custom sql scripts to get a REAL idea how your hardware performs. Note that if you will run the slow query you posted like once a minute and roll it up or cache it then don't get too worried about it. Pay attention to the queries that will add up, in aggregate, to your greatest load. {noformat} Not a whole lot of opinions except that tuning a server requires diligent work and performance is something one must do some work to achieve, and cannot be had by simply throwing money at the problem. h4. 20 May 2013 A few more responses over the weekend: {noformat} From: Greg Smith Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 7:44:16 PM PDT To: Mike McCann Cc: pgsql-performance@postgresql.org, Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On 5/13/13 6:36 PM, Mike McCann wrote: > stoqs_march2013_s=# explain analyze select * from > stoqs_measuredparameter order by datavalue; > > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual > time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 > rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > I tried changing random_page_cost to from 4 to 1 and saw no change. Have you tried putting an index by datavalue on this table? Once you've done that, then changing random_page_cost will make using that index look less expensive. Sorting chews through a good bit of CPU time, and that's where all of your runtime is being spent at--once you increase work_mem up very high that is. > I'm wondering now what changes might get this query to run in less than > one second. If all the data is in memory, then will faster CPU and > memory be the things that help? You're trying to fix a fundamental design issue with hardware. That usually doesn't go well. Once you get a box big enough to hold the whole database in RAM, beyond that the differences between server systems are relatively small. -- Greg Smith 2ndQuadrant US greg@2ndQuadrant.com Baltimore, MD PostgreSQL Training, Services, and 24x7 Support www.2ndQuadrant.com {noformat} and {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 8:57:32 PM PDT To: Greg Smith Cc: Mike McCann , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden , Jeff Janes On Sun, May 19, 2013 at 8:44 PM, Greg Smith wrote: >On 5/13/13 6:36 PM, Mike McCann wrote: >> >> stoqs_march2013_s=# explain analyze select * from >> stoqs_measuredparameter order by datavalue; >> >> QUERY PLAN >> >>------------------------------------------------------------------------------------------------------------------------------------------ >> Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual >> time=2503.078..2937.130 rows=3381814 loops=1) >> Sort Key: datavalue >> Sort Method: quicksort Memory: 362509kB >> -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 >> rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) >> Total runtime: 3094.601 ms >> (5 rows) >> >> I tried changing random_page_cost to from 4 to 1 and saw no change. > > > Have you tried putting an index by datavalue on this table? Once you've > done that, then changing random_page_cost will make using that index look > less expensive. Sorting chews through a good bit of CPU time, and that's > where all of your runtime is being spent at--once you increase work_mem up > very high that is. This++ plus cluster on that index if you can. {noformat} h4. 2 July 2013 Decision made to purchase server with FusionIO card. Server is now set up on the network with name "kraken". Here is the [confluence page for kraken's configuration settings| PostgreSQL Server kraken setup]. Setting up ability to run pgbench-tools on all the various servers. This can be done per user account by first installing from git and following the instructions in README.rst: {noformat} <108 pgtest.shore.mbari.org /mccann/dev> git clone git://git.postgresql.org/git/pgbench-tools.git Initialized empty Git repository in /u/mccann/dev/pgbench-tools/.git/ remote: Counting objects: 108, done. remote: Compressing objects: 100% (106/106), done. remote: Total 108 (delta 54), reused 0 (delta 0) Receiving objects: 100% (108/108), 24.61 KiB, done. Resolving deltas: 100% (54/54), done. {noformat}]]> 21365295 21398020 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 h4. 16 May 2013 Another post to the pgsql-perform mail list asking for opinions on a more conventional server configuration: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 3:36:03 PM PDT To: pgsql-performance@postgresql.org Cc: Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On May 7, 2013, at 4:21 PM, Jeff Janes wrote: On Thu, May 2, 2013 at 6:35 PM, Scott Marlowe wrote: On Thu, May 2, 2013 at 5:11 PM, Mike McCann wrote: > Hello, > > We are in the fortunate situation of having more money than time to help > solve our PostgreSQL 9.1 performance problem. > > Our server hosts databases that are about 1 GB in size with the largest > tables having order 10 million 20-byte indexed records. The data are loaded > once and then read from a web app and other client programs. Some of the > queries execute ORDER BY on the results. There are typically less than a > dozen read-only concurrent connections to any one database. I wouldn't count on this being a problem that can be fixed merely by throwing money at it. How many rows does any one of these queries need to access and then ORDER BY? ... > > HP ProLiant DL360p Gen 8 > Dual Intel Xeon 2.4GHz 4-core E5-2609 CPUs > 64GB RAM > 2x146GB 15K SAS hard drives > 3x200GB SATA SLC SSDs > + the usual accessories (optical drive, rail kit, dual power supplies) If your DB is 1G, and will grow to 10G then the IO shouldn't be any problem, as the whole db should be cached in memory. But it can take a surprisingly long time to get it cached in the first place, from a cold start. If that is the problem, pg_prewarm could help. Cheers, Jeff Thank you everyone for your suggestions. It's clear that our current read performance was not limited by hardware. An 'explain analyze' for a sample query is: stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=541002.15..549456.68 rows=3381814 width=20) (actual time=6254.780..7244.074 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..354.385 rows=3381814 loops=1) Total runtime: 7425.854 ms (5 rows) Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? We have an alternate (a bit more conventional) server configuration that we are considering: HP ProLiant DL360p Gen 8 Dual Intel Xeon 3.3GHz 4-core E5-2643 CPUs 128GB PC3-12800 RAM 16x146GB 15K SAS hard drives HP Smart Array P822/2GB FBWC controller + P420i w/ 2GB FBWC + the usual accessories (optical drive, rail kit, dual power supplies) All suggestions welcomed! -Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} The responses: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 4:58:17 PM PDT To: Jeff Janes Cc: "pgsql-performance@postgresql.org" , Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden On May 13, 2013, at 4:24 PM, Jeff Janes wrote: > On Mon, May 13, 2013 at 3:36 PM, Mike McCann wrote: > > Increasing work_mem to 355 MB improves the performance by a factor of 2: > > stoqs_march2013_s=# set work_mem='355MB'; > SET > stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > > I tried changing random_page_cost to from 4 to 1 and saw no change. > > I'm wondering now what changes might get this query to run in less than one second. I think you are worrying about the wrong thing here. What is a web app going to do with 3,381,814 rows, once it obtains them? Your current testing is not testing the time it takes to stream that data to the client, or for the client to do something meaningful with that data. If you only plan to actually fetch a few dozen of those rows, then you probably need to incorporate that into your test, either by using a LIMIT, or by using a mock-up of the actual application to do some timings. Also, what is the type and collation of the column you are sorting on? non-'C' collations of text columns sort about 3 times slower than 'C' collation does. If all the data is in memory, then will faster CPU and memory be the things that help? Yes, those would help (it is not clear to me which of the two would help more), but I think you need to rethink your design of sending the entire database table to the application server for each page-view. Cheers, Jeff Hi Jeff, The datavalue column is double precision: stoqs_march2013_s=# \d+ stoqs_measuredparameter Table "public.stoqs_measuredparameter" Column | Type | Modifiers | Storage | Description ----------------+------------------+----------------------------------------------------------------------+---------+------------- id | integer | not null default nextval('stoqs_measuredparameter_id_seq'::regclass) | plain | measurement_id | integer | not null | plain | parameter_id | integer | not null | plain | datavalue | double precision | not null | plain | Indexes: "stoqs_measuredparameter_pkey" PRIMARY KEY, btree (id) "stoqs_measuredparameter_measurement_id_parameter_id_key" UNIQUE CONSTRAINT, btree (measurement_id, parameter_id) "stoqs_measuredparameter_datavalue" btree (datavalue) "stoqs_measuredparameter_measurement_id" btree (measurement_id) "stoqs_measuredparameter_parameter_id" btree (parameter_id) Foreign-key constraints: "stoqs_measuredparameter_measurement_id_fkey" FOREIGN KEY (measurement_id) REFERENCES stoqs_measurement(id) DEFERRABLE INITIALLY DEFERRED "stoqs_measuredparameter_parameter_id_fkey" FOREIGN KEY (parameter_id) REFERENCES stoqs_parameter(id) DEFERRABLE INITIALLY DEFERRED Has OIDs: no Thanks for the suggestion and advice to examine the web app performance. We've actually taken quite a few steps to optimize how the web app works. The example query I provided is a simple worst-case one that we can use to help us decide on the proper hardware. An actual query performed by the web app is: stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_march2013_s-# stoqs_parameter.name AS parameter__name, stoqs_march2013_s-# stoqs_parameter.standard_name AS parameter__standard_name, stoqs_march2013_s-# stoqs_measurement.depth AS measurement__depth, stoqs_march2013_s-# stoqs_measurement.geom AS measurement__geom, stoqs_march2013_s-# stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_march2013_s-# stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_march2013_s-# stoqs_measuredparameter.datavalue AS datavalue, stoqs_march2013_s-# stoqs_parameter.units AS parameter__units stoqs_march2013_s-# FROM stoqs_parameter p1, stoqs_march2013_s-# stoqs_measuredparameter stoqs_march2013_s-# INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) stoqs_march2013_s-# INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) stoqs_march2013_s-# INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) stoqs_march2013_s-# INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) stoqs_march2013_s-# INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) stoqs_march2013_s-# INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id stoqs_march2013_s-# WHERE (p1.name = 'sea_water_sigma_t') stoqs_march2013_s-# AND (mp1.datavalue > 25.19) stoqs_march2013_s-# AND (mp1.datavalue < 26.01) stoqs_march2013_s-# AND (mp1.parameter_id = p1.id) stoqs_march2013_s-# AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' stoqs_march2013_s(# AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' stoqs_march2013_s(# AND stoqs_parameter.name IN ('fl700_uncorr') stoqs_march2013_s(# AND stoqs_measurement.depth >= -1.88 stoqs_march2013_s(# AND stoqs_platform.name IN ('dorado') stoqs_march2013_s(# AND stoqs_measurement.depth <= 83.57) stoqs_march2013_s-# ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=770.211..770.211 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=770.200..770.200 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=770.036..770.036 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=91.369..680.072 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=91.346..577.633 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=91.280..531.408 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=91.202..116.140 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.002..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=91.194..109.846 rows=20170 loop s=1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=90.794..90.794 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=62.769..62.769 rows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=27.412..2 7.412 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.008..0. 017 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.012..0.012 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.007..0.010 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.004..0.004 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.004..0.004 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.151..0.151 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.035..0.145 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.005..0.069 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.014..0.014 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.008..0.012 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 770.445 ms (42 rows) We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Thanks, Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} Another response: {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 6:09:58 PM PDT To: Mike McCann Cc: Jeff Janes , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden On Mon, May 13, 2013 at 5:58 PM, Mike McCann wrote: > We assume that steps taken to improve the worst-case query scenario will > also improve these kind of queries. If anything above pops out as needing > better planning please let us know that too! Bad assumption. If your real workload will be queries like the one here that takes 700 ms, but you'll be running 10,000 of them a second, you're tuning / hardware choices are going to be much different then if your query is going to be the previous 7 second one. Use realistic queries, not ones that are nothing like what your real ones will be. then use pgbench and its ability to run custom sql scripts to get a REAL idea how your hardware performs. Note that if you will run the slow query you posted like once a minute and roll it up or cache it then don't get too worried about it. Pay attention to the queries that will add up, in aggregate, to your greatest load. {noformat} Not a whole lot of opinions except that tuning a server requires diligent work and performance is something one must do some work to achieve, and cannot be had by simply throwing money at the problem. h4. 20 May 2013 A few more responses over the weekend: {noformat} From: Greg Smith Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 7:44:16 PM PDT To: Mike McCann Cc: pgsql-performance@postgresql.org, Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On 5/13/13 6:36 PM, Mike McCann wrote: > stoqs_march2013_s=# explain analyze select * from > stoqs_measuredparameter order by datavalue; > > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual > time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 > rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > I tried changing random_page_cost to from 4 to 1 and saw no change. Have you tried putting an index by datavalue on this table? Once you've done that, then changing random_page_cost will make using that index look less expensive. Sorting chews through a good bit of CPU time, and that's where all of your runtime is being spent at--once you increase work_mem up very high that is. > I'm wondering now what changes might get this query to run in less than > one second. If all the data is in memory, then will faster CPU and > memory be the things that help? You're trying to fix a fundamental design issue with hardware. That usually doesn't go well. Once you get a box big enough to hold the whole database in RAM, beyond that the differences between server systems are relatively small. -- Greg Smith 2ndQuadrant US greg@2ndQuadrant.com Baltimore, MD PostgreSQL Training, Services, and 24x7 Support www.2ndQuadrant.com {noformat} and {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 8:57:32 PM PDT To: Greg Smith Cc: Mike McCann , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden , Jeff Janes On Sun, May 19, 2013 at 8:44 PM, Greg Smith wrote: >On 5/13/13 6:36 PM, Mike McCann wrote: >> >> stoqs_march2013_s=# explain analyze select * from >> stoqs_measuredparameter order by datavalue; >> >> QUERY PLAN >> >>------------------------------------------------------------------------------------------------------------------------------------------ >> Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual >> time=2503.078..2937.130 rows=3381814 loops=1) >> Sort Key: datavalue >> Sort Method: quicksort Memory: 362509kB >> -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 >> rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) >> Total runtime: 3094.601 ms >> (5 rows) >> >> I tried changing random_page_cost to from 4 to 1 and saw no change. > > > Have you tried putting an index by datavalue on this table? Once you've > done that, then changing random_page_cost will make using that index look > less expensive. Sorting chews through a good bit of CPU time, and that's > where all of your runtime is being spent at--once you increase work_mem up > very high that is. This++ plus cluster on that index if you can. {noformat} ]]> 21365293 21398034 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 h4. 16 May 2013 Another post to the pgsql-perform mail list asking for opinions on a more conventional server configuration: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 3:36:03 PM PDT To: pgsql-performance@postgresql.org Cc: Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On May 7, 2013, at 4:21 PM, Jeff Janes wrote: On Thu, May 2, 2013 at 6:35 PM, Scott Marlowe wrote: On Thu, May 2, 2013 at 5:11 PM, Mike McCann wrote: > Hello, > > We are in the fortunate situation of having more money than time to help > solve our PostgreSQL 9.1 performance problem. > > Our server hosts databases that are about 1 GB in size with the largest > tables having order 10 million 20-byte indexed records. The data are loaded > once and then read from a web app and other client programs. Some of the > queries execute ORDER BY on the results. There are typically less than a > dozen read-only concurrent connections to any one database. I wouldn't count on this being a problem that can be fixed merely by throwing money at it. How many rows does any one of these queries need to access and then ORDER BY? ... > > HP ProLiant DL360p Gen 8 > Dual Intel Xeon 2.4GHz 4-core E5-2609 CPUs > 64GB RAM > 2x146GB 15K SAS hard drives > 3x200GB SATA SLC SSDs > + the usual accessories (optical drive, rail kit, dual power supplies) If your DB is 1G, and will grow to 10G then the IO shouldn't be any problem, as the whole db should be cached in memory. But it can take a surprisingly long time to get it cached in the first place, from a cold start. If that is the problem, pg_prewarm could help. Cheers, Jeff Thank you everyone for your suggestions. It's clear that our current read performance was not limited by hardware. An 'explain analyze' for a sample query is: stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=541002.15..549456.68 rows=3381814 width=20) (actual time=6254.780..7244.074 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..354.385 rows=3381814 loops=1) Total runtime: 7425.854 ms (5 rows) Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? We have an alternate (a bit more conventional) server configuration that we are considering: HP ProLiant DL360p Gen 8 Dual Intel Xeon 3.3GHz 4-core E5-2643 CPUs 128GB PC3-12800 RAM 16x146GB 15K SAS hard drives HP Smart Array P822/2GB FBWC controller + P420i w/ 2GB FBWC + the usual accessories (optical drive, rail kit, dual power supplies) All suggestions welcomed! -Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} The responses: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 4:58:17 PM PDT To: Jeff Janes Cc: "pgsql-performance@postgresql.org" , Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden On May 13, 2013, at 4:24 PM, Jeff Janes wrote: > On Mon, May 13, 2013 at 3:36 PM, Mike McCann wrote: > > Increasing work_mem to 355 MB improves the performance by a factor of 2: > > stoqs_march2013_s=# set work_mem='355MB'; > SET > stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > > I tried changing random_page_cost to from 4 to 1 and saw no change. > > I'm wondering now what changes might get this query to run in less than one second. I think you are worrying about the wrong thing here. What is a web app going to do with 3,381,814 rows, once it obtains them? Your current testing is not testing the time it takes to stream that data to the client, or for the client to do something meaningful with that data. If you only plan to actually fetch a few dozen of those rows, then you probably need to incorporate that into your test, either by using a LIMIT, or by using a mock-up of the actual application to do some timings. Also, what is the type and collation of the column you are sorting on? non-'C' collations of text columns sort about 3 times slower than 'C' collation does. If all the data is in memory, then will faster CPU and memory be the things that help? Yes, those would help (it is not clear to me which of the two would help more), but I think you need to rethink your design of sending the entire database table to the application server for each page-view. Cheers, Jeff Hi Jeff, The datavalue column is double precision: stoqs_march2013_s=# \d+ stoqs_measuredparameter Table "public.stoqs_measuredparameter" Column | Type | Modifiers | Storage | Description ----------------+------------------+----------------------------------------------------------------------+---------+------------- id | integer | not null default nextval('stoqs_measuredparameter_id_seq'::regclass) | plain | measurement_id | integer | not null | plain | parameter_id | integer | not null | plain | datavalue | double precision | not null | plain | Indexes: "stoqs_measuredparameter_pkey" PRIMARY KEY, btree (id) "stoqs_measuredparameter_measurement_id_parameter_id_key" UNIQUE CONSTRAINT, btree (measurement_id, parameter_id) "stoqs_measuredparameter_datavalue" btree (datavalue) "stoqs_measuredparameter_measurement_id" btree (measurement_id) "stoqs_measuredparameter_parameter_id" btree (parameter_id) Foreign-key constraints: "stoqs_measuredparameter_measurement_id_fkey" FOREIGN KEY (measurement_id) REFERENCES stoqs_measurement(id) DEFERRABLE INITIALLY DEFERRED "stoqs_measuredparameter_parameter_id_fkey" FOREIGN KEY (parameter_id) REFERENCES stoqs_parameter(id) DEFERRABLE INITIALLY DEFERRED Has OIDs: no Thanks for the suggestion and advice to examine the web app performance. We've actually taken quite a few steps to optimize how the web app works. The example query I provided is a simple worst-case one that we can use to help us decide on the proper hardware. An actual query performed by the web app is: stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_march2013_s-# stoqs_parameter.name AS parameter__name, stoqs_march2013_s-# stoqs_parameter.standard_name AS parameter__standard_name, stoqs_march2013_s-# stoqs_measurement.depth AS measurement__depth, stoqs_march2013_s-# stoqs_measurement.geom AS measurement__geom, stoqs_march2013_s-# stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_march2013_s-# stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_march2013_s-# stoqs_measuredparameter.datavalue AS datavalue, stoqs_march2013_s-# stoqs_parameter.units AS parameter__units stoqs_march2013_s-# FROM stoqs_parameter p1, stoqs_march2013_s-# stoqs_measuredparameter stoqs_march2013_s-# INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) stoqs_march2013_s-# INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) stoqs_march2013_s-# INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) stoqs_march2013_s-# INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) stoqs_march2013_s-# INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) stoqs_march2013_s-# INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id stoqs_march2013_s-# WHERE (p1.name = 'sea_water_sigma_t') stoqs_march2013_s-# AND (mp1.datavalue > 25.19) stoqs_march2013_s-# AND (mp1.datavalue < 26.01) stoqs_march2013_s-# AND (mp1.parameter_id = p1.id) stoqs_march2013_s-# AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' stoqs_march2013_s(# AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' stoqs_march2013_s(# AND stoqs_parameter.name IN ('fl700_uncorr') stoqs_march2013_s(# AND stoqs_measurement.depth >= -1.88 stoqs_march2013_s(# AND stoqs_platform.name IN ('dorado') stoqs_march2013_s(# AND stoqs_measurement.depth <= 83.57) stoqs_march2013_s-# ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=770.211..770.211 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=770.200..770.200 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=770.036..770.036 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=91.369..680.072 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=91.346..577.633 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=91.280..531.408 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=91.202..116.140 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.002..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=91.194..109.846 rows=20170 loop s=1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=90.794..90.794 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=62.769..62.769 rows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=27.412..2 7.412 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.008..0. 017 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.012..0.012 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.007..0.010 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.004..0.004 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.004..0.004 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.151..0.151 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.035..0.145 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.005..0.069 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.014..0.014 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.008..0.012 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 770.445 ms (42 rows) We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Thanks, Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} Another response: {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 6:09:58 PM PDT To: Mike McCann Cc: Jeff Janes , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden On Mon, May 13, 2013 at 5:58 PM, Mike McCann wrote: > We assume that steps taken to improve the worst-case query scenario will > also improve these kind of queries. If anything above pops out as needing > better planning please let us know that too! Bad assumption. If your real workload will be queries like the one here that takes 700 ms, but you'll be running 10,000 of them a second, you're tuning / hardware choices are going to be much different then if your query is going to be the previous 7 second one. Use realistic queries, not ones that are nothing like what your real ones will be. then use pgbench and its ability to run custom sql scripts to get a REAL idea how your hardware performs. Note that if you will run the slow query you posted like once a minute and roll it up or cache it then don't get too worried about it. Pay attention to the queries that will add up, in aggregate, to your greatest load. {noformat} Not a whole lot of opinions except that tuning a server requires diligent work and performance is something one must do some work to achieve, and cannot be had by simply throwing money at the problem. h4. 20 May 2013 A few more responses over the weekend: {noformat} From: Greg Smith Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 7:44:16 PM PDT To: Mike McCann Cc: pgsql-performance@postgresql.org, Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On 5/13/13 6:36 PM, Mike McCann wrote: > stoqs_march2013_s=# explain analyze select * from > stoqs_measuredparameter order by datavalue; > > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual > time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 > rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > I tried changing random_page_cost to from 4 to 1 and saw no change. Have you tried putting an index by datavalue on this table? Once you've done that, then changing random_page_cost will make using that index look less expensive. Sorting chews through a good bit of CPU time, and that's where all of your runtime is being spent at--once you increase work_mem up very high that is. > I'm wondering now what changes might get this query to run in less than > one second. If all the data is in memory, then will faster CPU and > memory be the things that help? You're trying to fix a fundamental design issue with hardware. That usually doesn't go well. Once you get a box big enough to hold the whole database in RAM, beyond that the differences between server systems are relatively small. -- Greg Smith 2ndQuadrant US greg@2ndQuadrant.com Baltimore, MD PostgreSQL Training, Services, and 24x7 Support www.2ndQuadrant.com {noformat} and {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 8:57:32 PM PDT To: Greg Smith Cc: Mike McCann , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden , Jeff Janes On Sun, May 19, 2013 at 8:44 PM, Greg Smith wrote: >On 5/13/13 6:36 PM, Mike McCann wrote: >> >> stoqs_march2013_s=# explain analyze select * from >> stoqs_measuredparameter order by datavalue; >> >> QUERY PLAN >> >>------------------------------------------------------------------------------------------------------------------------------------------ >> Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual >> time=2503.078..2937.130 rows=3381814 loops=1) >> Sort Key: datavalue >> Sort Method: quicksort Memory: 362509kB >> -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 >> rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) >> Total runtime: 3094.601 ms >> (5 rows) >> >> I tried changing random_page_cost to from 4 to 1 and saw no change. > > > Have you tried putting an index by datavalue on this table? Once you've > done that, then changing random_page_cost will make using that index look > less expensive. Sorting chews through a good bit of CPU time, and that's > where all of your runtime is being spent at--once you increase work_mem up > very high that is. This++ plus cluster on that index if you can. {noformat} h4. 2 July 2013 We made the decision made to purchase a server with a FusionIO card. Server is now set up on the network with name "kraken". Here is the [confluence page for kraken's configuration settings| PostgreSQL Server kraken setup]. Setting up ability to run pgbench-tools on all the various servers. This can be done per user account by first installing from git and following the instructions in README.rst: {noformat} <108 pgtest.shore.mbari.org /mccann/dev> git clone git://git.postgresql.org/git/pgbench-tools.git Initialized empty Git repository in /u/mccann/dev/pgbench-tools/.git/ remote: Counting objects: 108, done. remote: Compressing objects: 100% (106/106), done. remote: Total 108 (delta 54), reused 0 (delta 0) Receiving objects: 100% (108/108), 24.61 KiB, done. Resolving deltas: 100% (54/54), done. {noformat} The postgresql server on pgtest has higher security than the postgresql servers on our development VMs so the pgbench and results databases must be created by user postgres and then ownership granted to a shared account (stoqsadm) that we can use for running tests. Here's how the databases were created and set up: {noformat} su postgres psql # create database pgbench owner=stoqsadm; CREATE DATABASE # create database results owner=stoqsadm; CREATE DATABASE # \c pgbench You are now connected to database "pgbench" as user "postgres". pgbench=# grant all on all tables in schema public to stoqsadm; GRANT pgbench=# \c results You are now connected to database "results" as user "postgres". results=# grant all on all tables in schema public to stoqsadm; GRANT {noformat} Now we can run commands against these databases using the pgbench-tools commands: {noformat} <112 pgtest.shore.mbari.org /dev/pgbench-tools> psql -f init/resultdb.sql -d results -h localhost -U stoqsadm Password for user stoqsadm: psql:init/resultdb.sql:1: ERROR: table "timing" does not exist CREATE TABLE CREATE INDEX psql:init/resultdb.sql:12: ERROR: table "tests" does not exist psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_test_seq" for serial column "tests.test" psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_set_seq" for serial column "tests.set" CREATE TABLE psql:init/resultdb.sql:31: ERROR: table "testset" does not exist psql:init/resultdb.sql:36: NOTICE: CREATE TABLE will create implicit sequence "testset_set_seq" for serial column "testset.set" CREATE TABLE INSERT 0 1 psql:init/resultdb.sql:40: ERROR: table "test_bgwriter" does not exist CREATE TABLE <113 pgtest.shore.mbari.org /dev/pgbench-tools> psql -c "UPDATE testset SET info='Initial test set' WHERE set=1" -d results -h localhost -U stoqsadm Password for user stoqsadm: UPDATE 1 {noformat} I think those ERROR messages can be ignored because the following tests work. (The additional options "-h localhost -U stoqsadm" must be added to the example commands given in the pgbench-tools documentation.) The gnuplot package is needed for producing graphs of the results: {noformat} sudo -c "yum install gnuplot" {noformat} Edit the config file to reduce the number of scale and client runs so that we have a quick execution: {noformat} <114 pgtest.shore.mbari.org /dev/pgbench-tools> mv config config.130702 <115 pgtest.shore.mbari.org /dev/pgbench-tools> cp config.130702 config <116 pgtest.shore.mbari.org /dev/pgbench-tools> vi config <117 pgtest.shore.mbari.org /dev/pgbench-tools> diff config config.130702 42,45c42,43 < SCALES="1 10" < #SCALES="1 10 100 1000" < SETCLIENTS="1 2" < #SETCLIENTS="1 2 4 8 16 32" --- > SCALES="1 10 100 1000" > SETCLIENTS="1 2 4 8 16 32" {noformat} ]]> 21365307 9404576 9371826 21398032 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 h4. 16 May 2013 Another post to the pgsql-perform mail list asking for opinions on a more conventional server configuration: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 3:36:03 PM PDT To: pgsql-performance@postgresql.org Cc: Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On May 7, 2013, at 4:21 PM, Jeff Janes wrote: On Thu, May 2, 2013 at 6:35 PM, Scott Marlowe wrote: On Thu, May 2, 2013 at 5:11 PM, Mike McCann wrote: > Hello, > > We are in the fortunate situation of having more money than time to help > solve our PostgreSQL 9.1 performance problem. > > Our server hosts databases that are about 1 GB in size with the largest > tables having order 10 million 20-byte indexed records. The data are loaded > once and then read from a web app and other client programs. Some of the > queries execute ORDER BY on the results. There are typically less than a > dozen read-only concurrent connections to any one database. I wouldn't count on this being a problem that can be fixed merely by throwing money at it. How many rows does any one of these queries need to access and then ORDER BY? ... > > HP ProLiant DL360p Gen 8 > Dual Intel Xeon 2.4GHz 4-core E5-2609 CPUs > 64GB RAM > 2x146GB 15K SAS hard drives > 3x200GB SATA SLC SSDs > + the usual accessories (optical drive, rail kit, dual power supplies) If your DB is 1G, and will grow to 10G then the IO shouldn't be any problem, as the whole db should be cached in memory. But it can take a surprisingly long time to get it cached in the first place, from a cold start. If that is the problem, pg_prewarm could help. Cheers, Jeff Thank you everyone for your suggestions. It's clear that our current read performance was not limited by hardware. An 'explain analyze' for a sample query is: stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=541002.15..549456.68 rows=3381814 width=20) (actual time=6254.780..7244.074 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..354.385 rows=3381814 loops=1) Total runtime: 7425.854 ms (5 rows) Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? We have an alternate (a bit more conventional) server configuration that we are considering: HP ProLiant DL360p Gen 8 Dual Intel Xeon 3.3GHz 4-core E5-2643 CPUs 128GB PC3-12800 RAM 16x146GB 15K SAS hard drives HP Smart Array P822/2GB FBWC controller + P420i w/ 2GB FBWC + the usual accessories (optical drive, rail kit, dual power supplies) All suggestions welcomed! -Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} The responses: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 4:58:17 PM PDT To: Jeff Janes Cc: "pgsql-performance@postgresql.org" , Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden On May 13, 2013, at 4:24 PM, Jeff Janes wrote: > On Mon, May 13, 2013 at 3:36 PM, Mike McCann wrote: > > Increasing work_mem to 355 MB improves the performance by a factor of 2: > > stoqs_march2013_s=# set work_mem='355MB'; > SET > stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > > I tried changing random_page_cost to from 4 to 1 and saw no change. > > I'm wondering now what changes might get this query to run in less than one second. I think you are worrying about the wrong thing here. What is a web app going to do with 3,381,814 rows, once it obtains them? Your current testing is not testing the time it takes to stream that data to the client, or for the client to do something meaningful with that data. If you only plan to actually fetch a few dozen of those rows, then you probably need to incorporate that into your test, either by using a LIMIT, or by using a mock-up of the actual application to do some timings. Also, what is the type and collation of the column you are sorting on? non-'C' collations of text columns sort about 3 times slower than 'C' collation does. If all the data is in memory, then will faster CPU and memory be the things that help? Yes, those would help (it is not clear to me which of the two would help more), but I think you need to rethink your design of sending the entire database table to the application server for each page-view. Cheers, Jeff Hi Jeff, The datavalue column is double precision: stoqs_march2013_s=# \d+ stoqs_measuredparameter Table "public.stoqs_measuredparameter" Column | Type | Modifiers | Storage | Description ----------------+------------------+----------------------------------------------------------------------+---------+------------- id | integer | not null default nextval('stoqs_measuredparameter_id_seq'::regclass) | plain | measurement_id | integer | not null | plain | parameter_id | integer | not null | plain | datavalue | double precision | not null | plain | Indexes: "stoqs_measuredparameter_pkey" PRIMARY KEY, btree (id) "stoqs_measuredparameter_measurement_id_parameter_id_key" UNIQUE CONSTRAINT, btree (measurement_id, parameter_id) "stoqs_measuredparameter_datavalue" btree (datavalue) "stoqs_measuredparameter_measurement_id" btree (measurement_id) "stoqs_measuredparameter_parameter_id" btree (parameter_id) Foreign-key constraints: "stoqs_measuredparameter_measurement_id_fkey" FOREIGN KEY (measurement_id) REFERENCES stoqs_measurement(id) DEFERRABLE INITIALLY DEFERRED "stoqs_measuredparameter_parameter_id_fkey" FOREIGN KEY (parameter_id) REFERENCES stoqs_parameter(id) DEFERRABLE INITIALLY DEFERRED Has OIDs: no Thanks for the suggestion and advice to examine the web app performance. We've actually taken quite a few steps to optimize how the web app works. The example query I provided is a simple worst-case one that we can use to help us decide on the proper hardware. An actual query performed by the web app is: stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_march2013_s-# stoqs_parameter.name AS parameter__name, stoqs_march2013_s-# stoqs_parameter.standard_name AS parameter__standard_name, stoqs_march2013_s-# stoqs_measurement.depth AS measurement__depth, stoqs_march2013_s-# stoqs_measurement.geom AS measurement__geom, stoqs_march2013_s-# stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_march2013_s-# stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_march2013_s-# stoqs_measuredparameter.datavalue AS datavalue, stoqs_march2013_s-# stoqs_parameter.units AS parameter__units stoqs_march2013_s-# FROM stoqs_parameter p1, stoqs_march2013_s-# stoqs_measuredparameter stoqs_march2013_s-# INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) stoqs_march2013_s-# INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) stoqs_march2013_s-# INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) stoqs_march2013_s-# INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) stoqs_march2013_s-# INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) stoqs_march2013_s-# INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id stoqs_march2013_s-# WHERE (p1.name = 'sea_water_sigma_t') stoqs_march2013_s-# AND (mp1.datavalue > 25.19) stoqs_march2013_s-# AND (mp1.datavalue < 26.01) stoqs_march2013_s-# AND (mp1.parameter_id = p1.id) stoqs_march2013_s-# AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' stoqs_march2013_s(# AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' stoqs_march2013_s(# AND stoqs_parameter.name IN ('fl700_uncorr') stoqs_march2013_s(# AND stoqs_measurement.depth >= -1.88 stoqs_march2013_s(# AND stoqs_platform.name IN ('dorado') stoqs_march2013_s(# AND stoqs_measurement.depth <= 83.57) stoqs_march2013_s-# ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=770.211..770.211 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=770.200..770.200 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=770.036..770.036 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=91.369..680.072 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=91.346..577.633 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=91.280..531.408 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=91.202..116.140 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.002..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=91.194..109.846 rows=20170 loop s=1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=90.794..90.794 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=62.769..62.769 rows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=27.412..2 7.412 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.008..0. 017 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.012..0.012 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.007..0.010 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.004..0.004 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.004..0.004 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.151..0.151 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.035..0.145 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.005..0.069 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.014..0.014 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.008..0.012 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 770.445 ms (42 rows) We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Thanks, Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} Another response: {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 6:09:58 PM PDT To: Mike McCann Cc: Jeff Janes , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden On Mon, May 13, 2013 at 5:58 PM, Mike McCann wrote: > We assume that steps taken to improve the worst-case query scenario will > also improve these kind of queries. If anything above pops out as needing > better planning please let us know that too! Bad assumption. If your real workload will be queries like the one here that takes 700 ms, but you'll be running 10,000 of them a second, you're tuning / hardware choices are going to be much different then if your query is going to be the previous 7 second one. Use realistic queries, not ones that are nothing like what your real ones will be. then use pgbench and its ability to run custom sql scripts to get a REAL idea how your hardware performs. Note that if you will run the slow query you posted like once a minute and roll it up or cache it then don't get too worried about it. Pay attention to the queries that will add up, in aggregate, to your greatest load. {noformat} Not a whole lot of opinions except that tuning a server requires diligent work and performance is something one must do some work to achieve, and cannot be had by simply throwing money at the problem. h4. 20 May 2013 A few more responses over the weekend: {noformat} From: Greg Smith Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 7:44:16 PM PDT To: Mike McCann Cc: pgsql-performance@postgresql.org, Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On 5/13/13 6:36 PM, Mike McCann wrote: > stoqs_march2013_s=# explain analyze select * from > stoqs_measuredparameter order by datavalue; > > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual > time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 > rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > I tried changing random_page_cost to from 4 to 1 and saw no change. Have you tried putting an index by datavalue on this table? Once you've done that, then changing random_page_cost will make using that index look less expensive. Sorting chews through a good bit of CPU time, and that's where all of your runtime is being spent at--once you increase work_mem up very high that is. > I'm wondering now what changes might get this query to run in less than > one second. If all the data is in memory, then will faster CPU and > memory be the things that help? You're trying to fix a fundamental design issue with hardware. That usually doesn't go well. Once you get a box big enough to hold the whole database in RAM, beyond that the differences between server systems are relatively small. -- Greg Smith 2ndQuadrant US greg@2ndQuadrant.com Baltimore, MD PostgreSQL Training, Services, and 24x7 Support www.2ndQuadrant.com {noformat} and {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 8:57:32 PM PDT To: Greg Smith Cc: Mike McCann , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden , Jeff Janes On Sun, May 19, 2013 at 8:44 PM, Greg Smith wrote: >On 5/13/13 6:36 PM, Mike McCann wrote: >> >> stoqs_march2013_s=# explain analyze select * from >> stoqs_measuredparameter order by datavalue; >> >> QUERY PLAN >> >>------------------------------------------------------------------------------------------------------------------------------------------ >> Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual >> time=2503.078..2937.130 rows=3381814 loops=1) >> Sort Key: datavalue >> Sort Method: quicksort Memory: 362509kB >> -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 >> rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) >> Total runtime: 3094.601 ms >> (5 rows) >> >> I tried changing random_page_cost to from 4 to 1 and saw no change. > > > Have you tried putting an index by datavalue on this table? Once you've > done that, then changing random_page_cost will make using that index look > less expensive. Sorting chews through a good bit of CPU time, and that's > where all of your runtime is being spent at--once you increase work_mem up > very high that is. This++ plus cluster on that index if you can. {noformat} h4. 2 July 2013 We made the decision made to purchase a server with a FusionIO card. Server is now set up on the network with name "kraken". Here is the [confluence page for kraken's configuration settings| PostgreSQL Server kraken setup]. Setting up ability to run pgbench-tools on all the various servers. This can be done per user account by first installing from git and following the instructions in README.rst: {noformat} <108 pgtest.shore.mbari.org /mccann/dev> git clone git://git.postgresql.org/git/pgbench-tools.git Initialized empty Git repository in /u/mccann/dev/pgbench-tools/.git/ remote: Counting objects: 108, done. remote: Compressing objects: 100% (106/106), done. remote: Total 108 (delta 54), reused 0 (delta 0) Receiving objects: 100% (108/108), 24.61 KiB, done. Resolving deltas: 100% (54/54), done. {noformat} The postgresql server on pgtest has higher security than the postgresql servers on our development VMs so the pgbench and results databases must be created by user postgres and then ownership granted to a shared account (stoqsadm) that we can use for running tests. Here's how the databases were created and set up: {noformat} su postgres psql # create database pgbench owner=stoqsadm; CREATE DATABASE # create database results owner=stoqsadm; CREATE DATABASE # \c pgbench You are now connected to database "pgbench" as user "postgres". pgbench=# grant all on all tables in schema public to stoqsadm; GRANT pgbench=# \c results You are now connected to database "results" as user "postgres". results=# grant all on all tables in schema public to stoqsadm; GRANT {noformat} Now we can run commands against these databases using the pgbench-tools commands: {noformat} <112 pgtest.shore.mbari.org /dev/pgbench-tools> psql -f init/resultdb.sql -d results -h localhost -U stoqsadm Password for user stoqsadm: psql:init/resultdb.sql:1: ERROR: table "timing" does not exist CREATE TABLE CREATE INDEX psql:init/resultdb.sql:12: ERROR: table "tests" does not exist psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_test_seq" for serial column "tests.test" psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_set_seq" for serial column "tests.set" CREATE TABLE psql:init/resultdb.sql:31: ERROR: table "testset" does not exist psql:init/resultdb.sql:36: NOTICE: CREATE TABLE will create implicit sequence "testset_set_seq" for serial column "testset.set" CREATE TABLE INSERT 0 1 psql:init/resultdb.sql:40: ERROR: table "test_bgwriter" does not exist CREATE TABLE <113 pgtest.shore.mbari.org /dev/pgbench-tools> psql -c "UPDATE testset SET info='Initial test set' WHERE set=1" -d results -h localhost -U stoqsadm Password for user stoqsadm: UPDATE 1 {noformat} I think those EEROR messages can be ignored because the following tests work. (The additional options "-h localhost -U stoqsadm" must be added to the example commands given in the pgbench-tools documentation.) Edit the config file to reduce the number of scale and client runs so that we have a quick execution: {noformat} <114 pgtest.shore.mbari.org /dev/pgbench-tools> mv config config.130702 <115 pgtest.shore.mbari.org /dev/pgbench-tools> cp config.130702 config <116 pgtest.shore.mbari.org /dev/pgbench-tools> vi config <117 pgtest.shore.mbari.org /dev/pgbench-tools> diff config config.130702 42,45c42,43 < SCALES="1 10" < #SCALES="1 10 100 1000" < SETCLIENTS="1 2" < #SETCLIENTS="1 2 4 8 16 32" --- > SCALES="1 10 100 1000" > SETCLIENTS="1 2 4 8 16 32" {noformat} ]]> 21365305 9404578 Customize... menu:\\]]> 9371828 21398030 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 h4. 16 May 2013 Another post to the pgsql-perform mail list asking for opinions on a more conventional server configuration: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 3:36:03 PM PDT To: pgsql-performance@postgresql.org Cc: Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On May 7, 2013, at 4:21 PM, Jeff Janes wrote: On Thu, May 2, 2013 at 6:35 PM, Scott Marlowe wrote: On Thu, May 2, 2013 at 5:11 PM, Mike McCann wrote: > Hello, > > We are in the fortunate situation of having more money than time to help > solve our PostgreSQL 9.1 performance problem. > > Our server hosts databases that are about 1 GB in size with the largest > tables having order 10 million 20-byte indexed records. The data are loaded > once and then read from a web app and other client programs. Some of the > queries execute ORDER BY on the results. There are typically less than a > dozen read-only concurrent connections to any one database. I wouldn't count on this being a problem that can be fixed merely by throwing money at it. How many rows does any one of these queries need to access and then ORDER BY? ... > > HP ProLiant DL360p Gen 8 > Dual Intel Xeon 2.4GHz 4-core E5-2609 CPUs > 64GB RAM > 2x146GB 15K SAS hard drives > 3x200GB SATA SLC SSDs > + the usual accessories (optical drive, rail kit, dual power supplies) If your DB is 1G, and will grow to 10G then the IO shouldn't be any problem, as the whole db should be cached in memory. But it can take a surprisingly long time to get it cached in the first place, from a cold start. If that is the problem, pg_prewarm could help. Cheers, Jeff Thank you everyone for your suggestions. It's clear that our current read performance was not limited by hardware. An 'explain analyze' for a sample query is: stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=541002.15..549456.68 rows=3381814 width=20) (actual time=6254.780..7244.074 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..354.385 rows=3381814 loops=1) Total runtime: 7425.854 ms (5 rows) Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? We have an alternate (a bit more conventional) server configuration that we are considering: HP ProLiant DL360p Gen 8 Dual Intel Xeon 3.3GHz 4-core E5-2643 CPUs 128GB PC3-12800 RAM 16x146GB 15K SAS hard drives HP Smart Array P822/2GB FBWC controller + P420i w/ 2GB FBWC + the usual accessories (optical drive, rail kit, dual power supplies) All suggestions welcomed! -Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} The responses: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 4:58:17 PM PDT To: Jeff Janes Cc: "pgsql-performance@postgresql.org" , Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden On May 13, 2013, at 4:24 PM, Jeff Janes wrote: > On Mon, May 13, 2013 at 3:36 PM, Mike McCann wrote: > > Increasing work_mem to 355 MB improves the performance by a factor of 2: > > stoqs_march2013_s=# set work_mem='355MB'; > SET > stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > > I tried changing random_page_cost to from 4 to 1 and saw no change. > > I'm wondering now what changes might get this query to run in less than one second. I think you are worrying about the wrong thing here. What is a web app going to do with 3,381,814 rows, once it obtains them? Your current testing is not testing the time it takes to stream that data to the client, or for the client to do something meaningful with that data. If you only plan to actually fetch a few dozen of those rows, then you probably need to incorporate that into your test, either by using a LIMIT, or by using a mock-up of the actual application to do some timings. Also, what is the type and collation of the column you are sorting on? non-'C' collations of text columns sort about 3 times slower than 'C' collation does. If all the data is in memory, then will faster CPU and memory be the things that help? Yes, those would help (it is not clear to me which of the two would help more), but I think you need to rethink your design of sending the entire database table to the application server for each page-view. Cheers, Jeff Hi Jeff, The datavalue column is double precision: stoqs_march2013_s=# \d+ stoqs_measuredparameter Table "public.stoqs_measuredparameter" Column | Type | Modifiers | Storage | Description ----------------+------------------+----------------------------------------------------------------------+---------+------------- id | integer | not null default nextval('stoqs_measuredparameter_id_seq'::regclass) | plain | measurement_id | integer | not null | plain | parameter_id | integer | not null | plain | datavalue | double precision | not null | plain | Indexes: "stoqs_measuredparameter_pkey" PRIMARY KEY, btree (id) "stoqs_measuredparameter_measurement_id_parameter_id_key" UNIQUE CONSTRAINT, btree (measurement_id, parameter_id) "stoqs_measuredparameter_datavalue" btree (datavalue) "stoqs_measuredparameter_measurement_id" btree (measurement_id) "stoqs_measuredparameter_parameter_id" btree (parameter_id) Foreign-key constraints: "stoqs_measuredparameter_measurement_id_fkey" FOREIGN KEY (measurement_id) REFERENCES stoqs_measurement(id) DEFERRABLE INITIALLY DEFERRED "stoqs_measuredparameter_parameter_id_fkey" FOREIGN KEY (parameter_id) REFERENCES stoqs_parameter(id) DEFERRABLE INITIALLY DEFERRED Has OIDs: no Thanks for the suggestion and advice to examine the web app performance. We've actually taken quite a few steps to optimize how the web app works. The example query I provided is a simple worst-case one that we can use to help us decide on the proper hardware. An actual query performed by the web app is: stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_march2013_s-# stoqs_parameter.name AS parameter__name, stoqs_march2013_s-# stoqs_parameter.standard_name AS parameter__standard_name, stoqs_march2013_s-# stoqs_measurement.depth AS measurement__depth, stoqs_march2013_s-# stoqs_measurement.geom AS measurement__geom, stoqs_march2013_s-# stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_march2013_s-# stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_march2013_s-# stoqs_measuredparameter.datavalue AS datavalue, stoqs_march2013_s-# stoqs_parameter.units AS parameter__units stoqs_march2013_s-# FROM stoqs_parameter p1, stoqs_march2013_s-# stoqs_measuredparameter stoqs_march2013_s-# INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) stoqs_march2013_s-# INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) stoqs_march2013_s-# INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) stoqs_march2013_s-# INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) stoqs_march2013_s-# INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) stoqs_march2013_s-# INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id stoqs_march2013_s-# WHERE (p1.name = 'sea_water_sigma_t') stoqs_march2013_s-# AND (mp1.datavalue > 25.19) stoqs_march2013_s-# AND (mp1.datavalue < 26.01) stoqs_march2013_s-# AND (mp1.parameter_id = p1.id) stoqs_march2013_s-# AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' stoqs_march2013_s(# AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' stoqs_march2013_s(# AND stoqs_parameter.name IN ('fl700_uncorr') stoqs_march2013_s(# AND stoqs_measurement.depth >= -1.88 stoqs_march2013_s(# AND stoqs_platform.name IN ('dorado') stoqs_march2013_s(# AND stoqs_measurement.depth <= 83.57) stoqs_march2013_s-# ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=770.211..770.211 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=770.200..770.200 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=770.036..770.036 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=91.369..680.072 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=91.346..577.633 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=91.280..531.408 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=91.202..116.140 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.002..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=91.194..109.846 rows=20170 loop s=1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=90.794..90.794 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=62.769..62.769 rows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=27.412..2 7.412 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.008..0. 017 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.012..0.012 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.007..0.010 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.004..0.004 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.004..0.004 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.151..0.151 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.035..0.145 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.005..0.069 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.014..0.014 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.008..0.012 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 770.445 ms (42 rows) We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Thanks, Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} Another response: {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 6:09:58 PM PDT To: Mike McCann Cc: Jeff Janes , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden On Mon, May 13, 2013 at 5:58 PM, Mike McCann wrote: > We assume that steps taken to improve the worst-case query scenario will > also improve these kind of queries. If anything above pops out as needing > better planning please let us know that too! Bad assumption. If your real workload will be queries like the one here that takes 700 ms, but you'll be running 10,000 of them a second, you're tuning / hardware choices are going to be much different then if your query is going to be the previous 7 second one. Use realistic queries, not ones that are nothing like what your real ones will be. then use pgbench and its ability to run custom sql scripts to get a REAL idea how your hardware performs. Note that if you will run the slow query you posted like once a minute and roll it up or cache it then don't get too worried about it. Pay attention to the queries that will add up, in aggregate, to your greatest load. {noformat} Not a whole lot of opinions except that tuning a server requires diligent work and performance is something one must do some work to achieve, and cannot be had by simply throwing money at the problem. h4. 20 May 2013 A few more responses over the weekend: {noformat} From: Greg Smith Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 7:44:16 PM PDT To: Mike McCann Cc: pgsql-performance@postgresql.org, Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On 5/13/13 6:36 PM, Mike McCann wrote: > stoqs_march2013_s=# explain analyze select * from > stoqs_measuredparameter order by datavalue; > > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual > time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 > rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > I tried changing random_page_cost to from 4 to 1 and saw no change. Have you tried putting an index by datavalue on this table? Once you've done that, then changing random_page_cost will make using that index look less expensive. Sorting chews through a good bit of CPU time, and that's where all of your runtime is being spent at--once you increase work_mem up very high that is. > I'm wondering now what changes might get this query to run in less than > one second. If all the data is in memory, then will faster CPU and > memory be the things that help? You're trying to fix a fundamental design issue with hardware. That usually doesn't go well. Once you get a box big enough to hold the whole database in RAM, beyond that the differences between server systems are relatively small. -- Greg Smith 2ndQuadrant US greg@2ndQuadrant.com Baltimore, MD PostgreSQL Training, Services, and 24x7 Support www.2ndQuadrant.com {noformat} and {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 8:57:32 PM PDT To: Greg Smith Cc: Mike McCann , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden , Jeff Janes On Sun, May 19, 2013 at 8:44 PM, Greg Smith wrote: >On 5/13/13 6:36 PM, Mike McCann wrote: >> >> stoqs_march2013_s=# explain analyze select * from >> stoqs_measuredparameter order by datavalue; >> >> QUERY PLAN >> >>------------------------------------------------------------------------------------------------------------------------------------------ >> Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual >> time=2503.078..2937.130 rows=3381814 loops=1) >> Sort Key: datavalue >> Sort Method: quicksort Memory: 362509kB >> -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 >> rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) >> Total runtime: 3094.601 ms >> (5 rows) >> >> I tried changing random_page_cost to from 4 to 1 and saw no change. > > > Have you tried putting an index by datavalue on this table? Once you've > done that, then changing random_page_cost will make using that index look > less expensive. Sorting chews through a good bit of CPU time, and that's > where all of your runtime is being spent at--once you increase work_mem up > very high that is. This++ plus cluster on that index if you can. {noformat} h4. 2 July 2013 We made the decision made to purchase a server with a FusionIO card. Server is now set up on the network with name "kraken". Here is the [confluence page for kraken's configuration settings| PostgreSQL Server kraken setup]. Setting up ability to run pgbench-tools on all the various servers. This can be done per user account by first installing from git and following the instructions in README.rst: {noformat} <108 pgtest.shore.mbari.org /mccann/dev> git clone git://git.postgresql.org/git/pgbench-tools.git Initialized empty Git repository in /u/mccann/dev/pgbench-tools/.git/ remote: Counting objects: 108, done. remote: Compressing objects: 100% (106/106), done. remote: Total 108 (delta 54), reused 0 (delta 0) Receiving objects: 100% (108/108), 24.61 KiB, done. Resolving deltas: 100% (54/54), done. {noformat} The postgresql server on pgtest has higher security than the postgresql servers on our development VMs so the pgbench and results databases must be created by user postgres and then ownership granted to a shared account (stoqsadm) that we can use for running tests. Here's how the databases were created and set up: {noformat} su postgres psql # create database pgbench owner=stoqsadm; CREATE DATABASE # create database results owner=stoqsadm; CREATE DATABASE # \c pgbench You are now connected to database "pgbench" as user "postgres". pgbench=# grant all on all tables in schema public to stoqsadm; GRANT pgbench=# \c results You are now connected to database "results" as user "postgres". results=# grant all on all tables in schema public to stoqsadm; GRANT {noformat} Now we can run commands against these databases using the pgbench-tools commands: {noformat} <112 pgtest.shore.mbari.org /dev/pgbench-tools> psql -f init/resultdb.sql -d results -h localhost -U stoqsadm Password for user stoqsadm: psql:init/resultdb.sql:1: ERROR: table "timing" does not exist CREATE TABLE CREATE INDEX psql:init/resultdb.sql:12: ERROR: table "tests" does not exist psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_test_seq" for serial column "tests.test" psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_set_seq" for serial column "tests.set" CREATE TABLE psql:init/resultdb.sql:31: ERROR: table "testset" does not exist psql:init/resultdb.sql:36: NOTICE: CREATE TABLE will create implicit sequence "testset_set_seq" for serial column "testset.set" CREATE TABLE INSERT 0 1 psql:init/resultdb.sql:40: ERROR: table "test_bgwriter" does not exist CREATE TABLE {noformat} The additional options "-h localhost -U stoqsadm" must be added to the example commands given in the pgbench-tools documentation. ]]> 21365303 9404572 9371822 10944976 10912221 21398028 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 h4. 16 May 2013 Another post to the pgsql-perform mail list asking for opinions on a more conventional server configuration: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 3:36:03 PM PDT To: pgsql-performance@postgresql.org Cc: Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On May 7, 2013, at 4:21 PM, Jeff Janes wrote: On Thu, May 2, 2013 at 6:35 PM, Scott Marlowe wrote: On Thu, May 2, 2013 at 5:11 PM, Mike McCann wrote: > Hello, > > We are in the fortunate situation of having more money than time to help > solve our PostgreSQL 9.1 performance problem. > > Our server hosts databases that are about 1 GB in size with the largest > tables having order 10 million 20-byte indexed records. The data are loaded > once and then read from a web app and other client programs. Some of the > queries execute ORDER BY on the results. There are typically less than a > dozen read-only concurrent connections to any one database. I wouldn't count on this being a problem that can be fixed merely by throwing money at it. How many rows does any one of these queries need to access and then ORDER BY? ... > > HP ProLiant DL360p Gen 8 > Dual Intel Xeon 2.4GHz 4-core E5-2609 CPUs > 64GB RAM > 2x146GB 15K SAS hard drives > 3x200GB SATA SLC SSDs > + the usual accessories (optical drive, rail kit, dual power supplies) If your DB is 1G, and will grow to 10G then the IO shouldn't be any problem, as the whole db should be cached in memory. But it can take a surprisingly long time to get it cached in the first place, from a cold start. If that is the problem, pg_prewarm could help. Cheers, Jeff Thank you everyone for your suggestions. It's clear that our current read performance was not limited by hardware. An 'explain analyze' for a sample query is: stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=541002.15..549456.68 rows=3381814 width=20) (actual time=6254.780..7244.074 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..354.385 rows=3381814 loops=1) Total runtime: 7425.854 ms (5 rows) Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? We have an alternate (a bit more conventional) server configuration that we are considering: HP ProLiant DL360p Gen 8 Dual Intel Xeon 3.3GHz 4-core E5-2643 CPUs 128GB PC3-12800 RAM 16x146GB 15K SAS hard drives HP Smart Array P822/2GB FBWC controller + P420i w/ 2GB FBWC + the usual accessories (optical drive, rail kit, dual power supplies) All suggestions welcomed! -Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} The responses: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 4:58:17 PM PDT To: Jeff Janes Cc: "pgsql-performance@postgresql.org" , Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden On May 13, 2013, at 4:24 PM, Jeff Janes wrote: > On Mon, May 13, 2013 at 3:36 PM, Mike McCann wrote: > > Increasing work_mem to 355 MB improves the performance by a factor of 2: > > stoqs_march2013_s=# set work_mem='355MB'; > SET > stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > > I tried changing random_page_cost to from 4 to 1 and saw no change. > > I'm wondering now what changes might get this query to run in less than one second. I think you are worrying about the wrong thing here. What is a web app going to do with 3,381,814 rows, once it obtains them? Your current testing is not testing the time it takes to stream that data to the client, or for the client to do something meaningful with that data. If you only plan to actually fetch a few dozen of those rows, then you probably need to incorporate that into your test, either by using a LIMIT, or by using a mock-up of the actual application to do some timings. Also, what is the type and collation of the column you are sorting on? non-'C' collations of text columns sort about 3 times slower than 'C' collation does. If all the data is in memory, then will faster CPU and memory be the things that help? Yes, those would help (it is not clear to me which of the two would help more), but I think you need to rethink your design of sending the entire database table to the application server for each page-view. Cheers, Jeff Hi Jeff, The datavalue column is double precision: stoqs_march2013_s=# \d+ stoqs_measuredparameter Table "public.stoqs_measuredparameter" Column | Type | Modifiers | Storage | Description ----------------+------------------+----------------------------------------------------------------------+---------+------------- id | integer | not null default nextval('stoqs_measuredparameter_id_seq'::regclass) | plain | measurement_id | integer | not null | plain | parameter_id | integer | not null | plain | datavalue | double precision | not null | plain | Indexes: "stoqs_measuredparameter_pkey" PRIMARY KEY, btree (id) "stoqs_measuredparameter_measurement_id_parameter_id_key" UNIQUE CONSTRAINT, btree (measurement_id, parameter_id) "stoqs_measuredparameter_datavalue" btree (datavalue) "stoqs_measuredparameter_measurement_id" btree (measurement_id) "stoqs_measuredparameter_parameter_id" btree (parameter_id) Foreign-key constraints: "stoqs_measuredparameter_measurement_id_fkey" FOREIGN KEY (measurement_id) REFERENCES stoqs_measurement(id) DEFERRABLE INITIALLY DEFERRED "stoqs_measuredparameter_parameter_id_fkey" FOREIGN KEY (parameter_id) REFERENCES stoqs_parameter(id) DEFERRABLE INITIALLY DEFERRED Has OIDs: no Thanks for the suggestion and advice to examine the web app performance. We've actually taken quite a few steps to optimize how the web app works. The example query I provided is a simple worst-case one that we can use to help us decide on the proper hardware. An actual query performed by the web app is: stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_march2013_s-# stoqs_parameter.name AS parameter__name, stoqs_march2013_s-# stoqs_parameter.standard_name AS parameter__standard_name, stoqs_march2013_s-# stoqs_measurement.depth AS measurement__depth, stoqs_march2013_s-# stoqs_measurement.geom AS measurement__geom, stoqs_march2013_s-# stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_march2013_s-# stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_march2013_s-# stoqs_measuredparameter.datavalue AS datavalue, stoqs_march2013_s-# stoqs_parameter.units AS parameter__units stoqs_march2013_s-# FROM stoqs_parameter p1, stoqs_march2013_s-# stoqs_measuredparameter stoqs_march2013_s-# INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) stoqs_march2013_s-# INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) stoqs_march2013_s-# INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) stoqs_march2013_s-# INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) stoqs_march2013_s-# INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) stoqs_march2013_s-# INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id stoqs_march2013_s-# WHERE (p1.name = 'sea_water_sigma_t') stoqs_march2013_s-# AND (mp1.datavalue > 25.19) stoqs_march2013_s-# AND (mp1.datavalue < 26.01) stoqs_march2013_s-# AND (mp1.parameter_id = p1.id) stoqs_march2013_s-# AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' stoqs_march2013_s(# AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' stoqs_march2013_s(# AND stoqs_parameter.name IN ('fl700_uncorr') stoqs_march2013_s(# AND stoqs_measurement.depth >= -1.88 stoqs_march2013_s(# AND stoqs_platform.name IN ('dorado') stoqs_march2013_s(# AND stoqs_measurement.depth <= 83.57) stoqs_march2013_s-# ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=770.211..770.211 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=770.200..770.200 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=770.036..770.036 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=91.369..680.072 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=91.346..577.633 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=91.280..531.408 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=91.202..116.140 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.002..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=91.194..109.846 rows=20170 loop s=1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=90.794..90.794 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=62.769..62.769 rows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=27.412..2 7.412 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.008..0. 017 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.012..0.012 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.007..0.010 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.004..0.004 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.004..0.004 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.151..0.151 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.035..0.145 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.005..0.069 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.014..0.014 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.008..0.012 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 770.445 ms (42 rows) We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Thanks, Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} Another response: {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 6:09:58 PM PDT To: Mike McCann Cc: Jeff Janes , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden On Mon, May 13, 2013 at 5:58 PM, Mike McCann wrote: > We assume that steps taken to improve the worst-case query scenario will > also improve these kind of queries. If anything above pops out as needing > better planning please let us know that too! Bad assumption. If your real workload will be queries like the one here that takes 700 ms, but you'll be running 10,000 of them a second, you're tuning / hardware choices are going to be much different then if your query is going to be the previous 7 second one. Use realistic queries, not ones that are nothing like what your real ones will be. then use pgbench and its ability to run custom sql scripts to get a REAL idea how your hardware performs. Note that if you will run the slow query you posted like once a minute and roll it up or cache it then don't get too worried about it. Pay attention to the queries that will add up, in aggregate, to your greatest load. {noformat} Not a whole lot of opinions except that tuning a server requires diligent work and performance is something one must do some work to achieve, and cannot be had by simply throwing money at the problem. h4. 20 May 2013 A few more responses over the weekend: {noformat} From: Greg Smith Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 7:44:16 PM PDT To: Mike McCann Cc: pgsql-performance@postgresql.org, Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On 5/13/13 6:36 PM, Mike McCann wrote: > stoqs_march2013_s=# explain analyze select * from > stoqs_measuredparameter order by datavalue; > > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual > time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 > rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > I tried changing random_page_cost to from 4 to 1 and saw no change. Have you tried putting an index by datavalue on this table? Once you've done that, then changing random_page_cost will make using that index look less expensive. Sorting chews through a good bit of CPU time, and that's where all of your runtime is being spent at--once you increase work_mem up very high that is. > I'm wondering now what changes might get this query to run in less than > one second. If all the data is in memory, then will faster CPU and > memory be the things that help? You're trying to fix a fundamental design issue with hardware. That usually doesn't go well. Once you get a box big enough to hold the whole database in RAM, beyond that the differences between server systems are relatively small. -- Greg Smith 2ndQuadrant US greg@2ndQuadrant.com Baltimore, MD PostgreSQL Training, Services, and 24x7 Support www.2ndQuadrant.com {noformat} and {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 8:57:32 PM PDT To: Greg Smith Cc: Mike McCann , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden , Jeff Janes On Sun, May 19, 2013 at 8:44 PM, Greg Smith wrote: >On 5/13/13 6:36 PM, Mike McCann wrote: >> >> stoqs_march2013_s=# explain analyze select * from >> stoqs_measuredparameter order by datavalue; >> >> QUERY PLAN >> >>------------------------------------------------------------------------------------------------------------------------------------------ >> Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual >> time=2503.078..2937.130 rows=3381814 loops=1) >> Sort Key: datavalue >> Sort Method: quicksort Memory: 362509kB >> -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 >> rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) >> Total runtime: 3094.601 ms >> (5 rows) >> >> I tried changing random_page_cost to from 4 to 1 and saw no change. > > > Have you tried putting an index by datavalue on this table? Once you've > done that, then changing random_page_cost will make using that index look > less expensive. Sorting chews through a good bit of CPU time, and that's > where all of your runtime is being spent at--once you increase work_mem up > very high that is. This++ plus cluster on that index if you can. {noformat} h4. 2 July 2013 Decision made to purchase server with FusionIO card. Server is now set up on the network with name "kraken". Here is the [confluence page for kraken's configuration settings| PostgreSQL Server kraken setup]. Setting up ability to run pgbench-tools on all the various servers. This can be done per user account by first installing from git and following the instructions in README.rst: {noformat} <108 pgtest.shore.mbari.org /mccann/dev> git clone git://git.postgresql.org/git/pgbench-tools.git Initialized empty Git repository in /u/mccann/dev/pgbench-tools/.git/ remote: Counting objects: 108, done. remote: Compressing objects: 100% (106/106), done. remote: Total 108 (delta 54), reused 0 (delta 0) Receiving objects: 100% (108/108), 24.61 KiB, done. Resolving deltas: 100% (54/54), done. {noformat} The postgresql server on pgtest has higher security than the postgresql servers on our development VMs so the pgbench and results databases must be created as user postgres and then ownership granted to a shared account (stoqsadm) that we can use for running tests. Here's how the databases were created and set up: {noformat} su postgres psql # create database pgbench owner=stoqsadm; CREATE DATABASE # create database results owner=stoqsadm; CREATE DATABASE # \c pgbench You are now connected to database "pgbench" as user "postgres". pgbench=# grant all on all tables in schema public to stoqsadm; GRANT pgbench=# \c results You are now connected to database "results" as user "postgres". results=# grant all on all tables in schema public to stoqsadm; GRANT {noformat} Now we can run commands against these databases using the pgbench-tools commands: {noformat} <112 pgtest.shore.mbari.org /dev/pgbench-tools> psql -f init/resultdb.sql -d results -h localhost -U stoqsadm Password for user stoqsadm: psql:init/resultdb.sql:1: ERROR: table "timing" does not exist CREATE TABLE CREATE INDEX psql:init/resultdb.sql:12: ERROR: table "tests" does not exist psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_test_seq" for serial column "tests.test" psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_set_seq" for serial column "tests.set" CREATE TABLE psql:init/resultdb.sql:31: ERROR: table "testset" does not exist psql:init/resultdb.sql:36: NOTICE: CREATE TABLE will create implicit sequence "testset_set_seq" for serial column "testset.set" CREATE TABLE INSERT 0 1 psql:init/resultdb.sql:40: ERROR: table "test_bgwriter" does not exist CREATE TABLE {noformat} The additional options "-h localhost -U stoqsadm" must be added to the example commands given in the pgbench-tools documentation. ]]> 21365301 9404585 Customize... menu (check the ArcMapNav box): \\ !ArcMabNav_fig1.png! Click the ArcMapNav Setup button (select the Ship and see that positions are being updated): !ArcMabNav_fig2.png!     Add the ROV and Ship Layers, and any other base layers you wish: !ArcMabNav_fig3.png!   Saving Markpoints (click on Mouse button or Target button): !ArcMabNav_fig4.png!\\ Getting Range and Bearing (click on Compas button): !ArcMabNav_fig5.png!  ]]> 9371835 9404584 Customize... menu (check the ArcMapNav box): \\ !ArcMabNav_fig1.png! Click the ArcMapNav Setup button (select the Ship and see that positions are being updated): !ArcMabNav_fig2.png!     Add the ROV and Ship Layers, and any other base layers you wish: !ArcMabNav_fig3.png!  ]]> 9371834 9404582 Customize... menu (check the ArcMapNav box): \\ !ArcMabNav_fig1.png! Click the ArcMapNav Setup button (select the Ship and see that positions are being updated): !ArcMabNav_fig2.png!     Add the ROV and Ship Layers: !ArcMabNav_fig3.png!   ]]> 9371832 11273169 11240432 9404580 Customize... menu (check the ArcMapNav box): \\ !ArcMabNav_fig1.png! Click the ArcMapNav Setup button (select the Ship and see that positions are being updated): !ArcMabNav_fig2.png! !ArcMabNav_fig3.png!       Add the ROV and Ship Layers:    ]]> 9371830 5407456 5374725 9404593 9371843 5407458 5374727 9404592 9371842 9404587 9371837 19074136 19041450 9404601 9371851 9404602 9371852 19074138 19041452 9404599 9371849 9404597 9371847 19074134 19041448 9404595 9371845 19074133 19041447 9404596 9371846 21398065 #checkpoint_segments = 3 # in logfile segments, min 1, 16MB each 180c180 < checkpoint_completion_target = 0.9 # checkpoint target duration, 0.0 - 1.0 --- > #checkpoint_completion_target = 0.5 # checkpoint target duration, 0.0 - 1.0 {noformat} Changed due to log {noformat} LOG: checkpoints are occurring too frequently (2 seconds apart) HINT: Consider increasing the configuration parameter "checkpoint_segments". {noformat}]]> 21365339 19074140 19041454 21398059 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 h4. 16 May 2013 Another post to the pgsql-perform mail list asking for opinions on a more conventional server configuration: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 3:36:03 PM PDT To: pgsql-performance@postgresql.org Cc: Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On May 7, 2013, at 4:21 PM, Jeff Janes wrote: On Thu, May 2, 2013 at 6:35 PM, Scott Marlowe wrote: On Thu, May 2, 2013 at 5:11 PM, Mike McCann wrote: > Hello, > > We are in the fortunate situation of having more money than time to help > solve our PostgreSQL 9.1 performance problem. > > Our server hosts databases that are about 1 GB in size with the largest > tables having order 10 million 20-byte indexed records. The data are loaded > once and then read from a web app and other client programs. Some of the > queries execute ORDER BY on the results. There are typically less than a > dozen read-only concurrent connections to any one database. I wouldn't count on this being a problem that can be fixed merely by throwing money at it. How many rows does any one of these queries need to access and then ORDER BY? ... > > HP ProLiant DL360p Gen 8 > Dual Intel Xeon 2.4GHz 4-core E5-2609 CPUs > 64GB RAM > 2x146GB 15K SAS hard drives > 3x200GB SATA SLC SSDs > + the usual accessories (optical drive, rail kit, dual power supplies) If your DB is 1G, and will grow to 10G then the IO shouldn't be any problem, as the whole db should be cached in memory. But it can take a surprisingly long time to get it cached in the first place, from a cold start. If that is the problem, pg_prewarm could help. Cheers, Jeff Thank you everyone for your suggestions. It's clear that our current read performance was not limited by hardware. An 'explain analyze' for a sample query is: stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=541002.15..549456.68 rows=3381814 width=20) (actual time=6254.780..7244.074 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..354.385 rows=3381814 loops=1) Total runtime: 7425.854 ms (5 rows) Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? We have an alternate (a bit more conventional) server configuration that we are considering: HP ProLiant DL360p Gen 8 Dual Intel Xeon 3.3GHz 4-core E5-2643 CPUs 128GB PC3-12800 RAM 16x146GB 15K SAS hard drives HP Smart Array P822/2GB FBWC controller + P420i w/ 2GB FBWC + the usual accessories (optical drive, rail kit, dual power supplies) All suggestions welcomed! -Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} The responses: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 4:58:17 PM PDT To: Jeff Janes Cc: "pgsql-performance@postgresql.org" , Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden On May 13, 2013, at 4:24 PM, Jeff Janes wrote: > On Mon, May 13, 2013 at 3:36 PM, Mike McCann wrote: > > Increasing work_mem to 355 MB improves the performance by a factor of 2: > > stoqs_march2013_s=# set work_mem='355MB'; > SET > stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > > I tried changing random_page_cost to from 4 to 1 and saw no change. > > I'm wondering now what changes might get this query to run in less than one second. I think you are worrying about the wrong thing here. What is a web app going to do with 3,381,814 rows, once it obtains them? Your current testing is not testing the time it takes to stream that data to the client, or for the client to do something meaningful with that data. If you only plan to actually fetch a few dozen of those rows, then you probably need to incorporate that into your test, either by using a LIMIT, or by using a mock-up of the actual application to do some timings. Also, what is the type and collation of the column you are sorting on? non-'C' collations of text columns sort about 3 times slower than 'C' collation does. If all the data is in memory, then will faster CPU and memory be the things that help? Yes, those would help (it is not clear to me which of the two would help more), but I think you need to rethink your design of sending the entire database table to the application server for each page-view. Cheers, Jeff Hi Jeff, The datavalue column is double precision: stoqs_march2013_s=# \d+ stoqs_measuredparameter Table "public.stoqs_measuredparameter" Column | Type | Modifiers | Storage | Description ----------------+------------------+----------------------------------------------------------------------+---------+------------- id | integer | not null default nextval('stoqs_measuredparameter_id_seq'::regclass) | plain | measurement_id | integer | not null | plain | parameter_id | integer | not null | plain | datavalue | double precision | not null | plain | Indexes: "stoqs_measuredparameter_pkey" PRIMARY KEY, btree (id) "stoqs_measuredparameter_measurement_id_parameter_id_key" UNIQUE CONSTRAINT, btree (measurement_id, parameter_id) "stoqs_measuredparameter_datavalue" btree (datavalue) "stoqs_measuredparameter_measurement_id" btree (measurement_id) "stoqs_measuredparameter_parameter_id" btree (parameter_id) Foreign-key constraints: "stoqs_measuredparameter_measurement_id_fkey" FOREIGN KEY (measurement_id) REFERENCES stoqs_measurement(id) DEFERRABLE INITIALLY DEFERRED "stoqs_measuredparameter_parameter_id_fkey" FOREIGN KEY (parameter_id) REFERENCES stoqs_parameter(id) DEFERRABLE INITIALLY DEFERRED Has OIDs: no Thanks for the suggestion and advice to examine the web app performance. We've actually taken quite a few steps to optimize how the web app works. The example query I provided is a simple worst-case one that we can use to help us decide on the proper hardware. An actual query performed by the web app is: stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_march2013_s-# stoqs_parameter.name AS parameter__name, stoqs_march2013_s-# stoqs_parameter.standard_name AS parameter__standard_name, stoqs_march2013_s-# stoqs_measurement.depth AS measurement__depth, stoqs_march2013_s-# stoqs_measurement.geom AS measurement__geom, stoqs_march2013_s-# stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_march2013_s-# stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_march2013_s-# stoqs_measuredparameter.datavalue AS datavalue, stoqs_march2013_s-# stoqs_parameter.units AS parameter__units stoqs_march2013_s-# FROM stoqs_parameter p1, stoqs_march2013_s-# stoqs_measuredparameter stoqs_march2013_s-# INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) stoqs_march2013_s-# INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) stoqs_march2013_s-# INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) stoqs_march2013_s-# INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) stoqs_march2013_s-# INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) stoqs_march2013_s-# INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id stoqs_march2013_s-# WHERE (p1.name = 'sea_water_sigma_t') stoqs_march2013_s-# AND (mp1.datavalue > 25.19) stoqs_march2013_s-# AND (mp1.datavalue < 26.01) stoqs_march2013_s-# AND (mp1.parameter_id = p1.id) stoqs_march2013_s-# AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' stoqs_march2013_s(# AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' stoqs_march2013_s(# AND stoqs_parameter.name IN ('fl700_uncorr') stoqs_march2013_s(# AND stoqs_measurement.depth >= -1.88 stoqs_march2013_s(# AND stoqs_platform.name IN ('dorado') stoqs_march2013_s(# AND stoqs_measurement.depth <= 83.57) stoqs_march2013_s-# ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=770.211..770.211 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=770.200..770.200 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=770.036..770.036 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=91.369..680.072 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=91.346..577.633 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=91.280..531.408 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=91.202..116.140 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.002..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=91.194..109.846 rows=20170 loop s=1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=90.794..90.794 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=62.769..62.769 rows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=27.412..2 7.412 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.008..0. 017 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.012..0.012 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.007..0.010 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.004..0.004 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.004..0.004 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.151..0.151 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.035..0.145 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.005..0.069 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.014..0.014 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.008..0.012 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 770.445 ms (42 rows) We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Thanks, Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} Another response: {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 6:09:58 PM PDT To: Mike McCann Cc: Jeff Janes , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden On Mon, May 13, 2013 at 5:58 PM, Mike McCann wrote: > We assume that steps taken to improve the worst-case query scenario will > also improve these kind of queries. If anything above pops out as needing > better planning please let us know that too! Bad assumption. If your real workload will be queries like the one here that takes 700 ms, but you'll be running 10,000 of them a second, you're tuning / hardware choices are going to be much different then if your query is going to be the previous 7 second one. Use realistic queries, not ones that are nothing like what your real ones will be. then use pgbench and its ability to run custom sql scripts to get a REAL idea how your hardware performs. Note that if you will run the slow query you posted like once a minute and roll it up or cache it then don't get too worried about it. Pay attention to the queries that will add up, in aggregate, to your greatest load. {noformat} Not a whole lot of opinions except that tuning a server requires diligent work and performance is something one must do some work to achieve, and cannot be had by simply throwing money at the problem. h4. 20 May 2013 A few more responses over the weekend: {noformat} From: Greg Smith Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 7:44:16 PM PDT To: Mike McCann Cc: pgsql-performance@postgresql.org, Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On 5/13/13 6:36 PM, Mike McCann wrote: > stoqs_march2013_s=# explain analyze select * from > stoqs_measuredparameter order by datavalue; > > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual > time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 > rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > I tried changing random_page_cost to from 4 to 1 and saw no change. Have you tried putting an index by datavalue on this table? Once you've done that, then changing random_page_cost will make using that index look less expensive. Sorting chews through a good bit of CPU time, and that's where all of your runtime is being spent at--once you increase work_mem up very high that is. > I'm wondering now what changes might get this query to run in less than > one second. If all the data is in memory, then will faster CPU and > memory be the things that help? You're trying to fix a fundamental design issue with hardware. That usually doesn't go well. Once you get a box big enough to hold the whole database in RAM, beyond that the differences between server systems are relatively small. -- Greg Smith 2ndQuadrant US greg@2ndQuadrant.com Baltimore, MD PostgreSQL Training, Services, and 24x7 Support www.2ndQuadrant.com {noformat} and {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 8:57:32 PM PDT To: Greg Smith Cc: Mike McCann , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden , Jeff Janes On Sun, May 19, 2013 at 8:44 PM, Greg Smith wrote: >On 5/13/13 6:36 PM, Mike McCann wrote: >> >> stoqs_march2013_s=# explain analyze select * from >> stoqs_measuredparameter order by datavalue; >> >> QUERY PLAN >> >>------------------------------------------------------------------------------------------------------------------------------------------ >> Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual >> time=2503.078..2937.130 rows=3381814 loops=1) >> Sort Key: datavalue >> Sort Method: quicksort Memory: 362509kB >> -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 >> rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) >> Total runtime: 3094.601 ms >> (5 rows) >> >> I tried changing random_page_cost to from 4 to 1 and saw no change. > > > Have you tried putting an index by datavalue on this table? Once you've > done that, then changing random_page_cost will make using that index look > less expensive. Sorting chews through a good bit of CPU time, and that's > where all of your runtime is being spent at--once you increase work_mem up > very high that is. This++ plus cluster on that index if you can. {noformat} h4. 2 July 2013 We made the decision made to purchase a server with a FusionIO card. Server is now set up on the network with name "kraken". Here is the [confluence page for kraken's configuration settings| PostgreSQL Server kraken setup]. Setting up ability to run pgbench-tools on all the various servers. This may be done per user account by first installing from git and following the instructions in README.rst; however, the scripts need to run commands requiring createdb and vacuum privileges, so it's best to run as user postgres. So, logging into pgtest as user postgres and getting things set up: {noformat} -bash-4.1$ pwd /var/lib/pgsql -bash-4.1$ mkdir dev -bash-4.1$ cd dev -bash-4.1$ git clone git://git.postgresql.org/git/pgbench-tools.git Initialized empty Git repository in /var/lib/pgsql/dev/pgbench-tools/.git/ remote: Counting objects: 108, done. remote: Compressing objects: 100% (106/106), done. remote: Total 108 (delta 54), reused 0 (delta 0) Receiving objects: 100% (108/108), 24.61 KiB, done. Resolving deltas: 100% (54/54), done. -bash-4.1$ createdb results -bash-4.1$ createdb pgbench -bash-4.1$ cd pgbench-tools/ -bash-4.1$ psql -f init/resultdb.sql -d results psql:init/resultdb.sql:1: ERROR: table "timing" does not exist CREATE TABLE CREATE INDEX psql:init/resultdb.sql:12: ERROR: table "tests" does not exist psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_test_seq" for serial column "tests.test" psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_set_seq" for serial column "tests.set" CREATE TABLE psql:init/resultdb.sql:31: ERROR: table "testset" does not exist psql:init/resultdb.sql:36: NOTICE: CREATE TABLE will create implicit sequence "testset_set_seq" for serial column "testset.set" CREATE TABLE INSERT 0 1 psql:init/resultdb.sql:40: ERROR: table "test_bgwriter" does not exist CREATE TABLE -bash-4.1$ psql -c "UPDATE testset SET info='Initial test set' WHERE set=1" -d results UPDATE 1 {noformat} I think those ERROR messages can be ignored because the following tests work. The gnuplot package is needed for producing graphs of the results: {noformat} sudo -c "yum install gnuplot" {noformat} Edit the config file with hard path to pgbench and reduced number of scale and client runs so that we have a quick execution to just test things: {noformat} -bash-4.1$ mv config config.130702 -bash-4.1$ cp config.130702 config -bash-4.1$ vi config -bash-4.1$ diff config.130702 config 8c8 < PGBENCHBIN=`which pgbench` --- > PGBENCHBIN=/usr/pgsql-9.1/bin/pgbench 42,44c42,47 < SCALES="1 10 100 1000" < SETCLIENTS="1 2 4 8 16 32" < SETTIMES=3 --- > SCALES="1" > SETCLIENTS="1" > SETTIMES=1 > #SCALES="1 10 100 1000" > #SETCLIENTS="1 2 4 8 16 32" > #SETTIMES=3 {noformat} Edit scripts to use "local socket" connection (removing "-h $TESTHOST") for the postgres user: {noformat} -bash-4.1$ diff runset.130702 runset 19c19 < TESTPSQL="psql -h $TESTHOST -U $TESTUSER -p $TESTPORT -d $TESTDB" --- > TESTPSQL="psql -U $TESTUSER -p $TESTPORT -d $TESTDB" 24c24 < $PGBENCHBIN -i -s $SCALE -h $TESTHOST -U $TESTUSER -p $TESTPORT $TESTDB --- > $PGBENCHBIN -i -s $SCALE -U $TESTUSER -p $TESTPORT $TESTDB -bash-4.1$ diff benchwarmer.130702 benchwarmer 84,85c84,85 < TESTPSQL="psql -h $TESTHOST -U $TESTUSER -p $TESTPORT -d $TESTDB" < RESULTPSQL="psql -h $RESULTHOST -U $RESULTUSER -p $RESULTPORT -d $RESULTDB" --- > TESTPSQL="psql -U $TESTUSER -p $TESTPORT -d $TESTDB" > RESULTPSQL="psql -U $RESULTUSER -p $RESULTPORT -d $RESULTDB" 158c158 < $PGBENCHBIN -f $BASEDIR/$TESTDIR/$SCRIPT -s $SCALE -l -n $TESTLEN -U $TESTUSER -h $TESTHOST -p $TESTPORT -c $CLIENTS $WORKERS $TESTDB > results.txt & --- > $PGBENCHBIN -f $BASEDIR/$TESTDIR/$SCRIPT -s $SCALE -l -n $TESTLEN -U $TESTUSER -p $TESTPORT -c $CLIENTS $WORKERS $TESTDB > results.txt & -bash-4.1$ diff webreport.130702 webreport 10c10 < RESULTPSQL="psql -h $RESULTHOST -U $RESULTUSER -p $RESULTPORT -d $RESULTDB" --- > RESULTPSQL="psql -U $RESULTUSER -p $RESULTPORT -d $RESULTDB" {noformat} And now a test run looks like: {noformat} -bash-4.1$ ./runset Removing old pgbench tables DROP TABLE VACUUM Creating new pgbench tables NOTICE: table "pgbench_branches" does not exist, skipping NOTICE: table "pgbench_tellers" does not exist, skipping NOTICE: table "pgbench_accounts" does not exist, skipping NOTICE: table "pgbench_history" does not exist, skipping creating tables... 10000 tuples done. 20000 tuples done. 30000 tuples done. 40000 tuples done. 50000 tuples done. 60000 tuples done. 70000 tuples done. 80000 tuples done. 90000 tuples done. 100000 tuples done. set primary key... NOTICE: ALTER TABLE / ADD PRIMARY KEY will create implicit index "pgbench_branches_pkey" for table "pgbench_branches" NOTICE: ALTER TABLE / ADD PRIMARY KEY will create implicit index "pgbench_tellers_pkey" for table "pgbench_tellers" NOTICE: ALTER TABLE / ADD PRIMARY KEY will create implicit index "pgbench_accounts_pkey" for table "pgbench_accounts" vacuum...done. Run set #1 of 1 with 1 clients scale=1 Running tests using: psql -U postgres -p 5432 -d pgbench Storing results using: psql -U postgres -p 5432 -d results Cleaning up database pgbench TRUNCATE TABLE VACUUM CHECKPOINT INSERT 0 1 This is test 3 Script select.sql executing for 1 concurrent users... UPDATE 1 transaction type: Custom query scaling factor: 1 query mode: simple number of clients: 1 number of threads: 1 duration: 60 s number of transactions actually processed: 547599 tps = 9126.622772 (including connections establishing) tps = 9127.071214 (excluding connections establishing) Worst latency results: 2018 2311 2370 3155 3476 Could not find/open font when opening font "arial", using internal non-scalable font Could not find/open font when opening font "arial", using internal non-scalable font Could not find/open font when opening font "arial", using internal non-scalable font Warning: empty x range [1:1], adjusting to [0.99:1.01] Warning: empty y range [9228:9228], adjusting to [9135.72:9320.28] Warning: empty y2 range [21:21], adjusting to [20.79:21.21] Could not find/open font when opening font "arial", using internal non-scalable font Warning: empty x range [1:1], adjusting to [0.99:1.01] Warning: empty y range [9228:9228], adjusting to [9135.72:9320.28] Could not find/open font when opening font "arial", using internal non-scalable font Warning: empty x range [1:1], adjusting to [0.99:1.01] Warning: empty y range [1:1], adjusting to [0.99:1.01] Warning: empty z range [9228:9228], adjusting to [9135.72:9320.28] {noformat} ]]> 21365333 9404606 9371856 19074142 19041456 21398061 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 h4. 16 May 2013 Another post to the pgsql-perform mail list asking for opinions on a more conventional server configuration: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 3:36:03 PM PDT To: pgsql-performance@postgresql.org Cc: Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On May 7, 2013, at 4:21 PM, Jeff Janes wrote: On Thu, May 2, 2013 at 6:35 PM, Scott Marlowe wrote: On Thu, May 2, 2013 at 5:11 PM, Mike McCann wrote: > Hello, > > We are in the fortunate situation of having more money than time to help > solve our PostgreSQL 9.1 performance problem. > > Our server hosts databases that are about 1 GB in size with the largest > tables having order 10 million 20-byte indexed records. The data are loaded > once and then read from a web app and other client programs. Some of the > queries execute ORDER BY on the results. There are typically less than a > dozen read-only concurrent connections to any one database. I wouldn't count on this being a problem that can be fixed merely by throwing money at it. How many rows does any one of these queries need to access and then ORDER BY? ... > > HP ProLiant DL360p Gen 8 > Dual Intel Xeon 2.4GHz 4-core E5-2609 CPUs > 64GB RAM > 2x146GB 15K SAS hard drives > 3x200GB SATA SLC SSDs > + the usual accessories (optical drive, rail kit, dual power supplies) If your DB is 1G, and will grow to 10G then the IO shouldn't be any problem, as the whole db should be cached in memory. But it can take a surprisingly long time to get it cached in the first place, from a cold start. If that is the problem, pg_prewarm could help. Cheers, Jeff Thank you everyone for your suggestions. It's clear that our current read performance was not limited by hardware. An 'explain analyze' for a sample query is: stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=541002.15..549456.68 rows=3381814 width=20) (actual time=6254.780..7244.074 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..354.385 rows=3381814 loops=1) Total runtime: 7425.854 ms (5 rows) Increasing work_mem to 355 MB improves the performance by a factor of 2: stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) Total runtime: 3094.601 ms (5 rows) I tried changing random_page_cost to from 4 to 1 and saw no change. I'm wondering now what changes might get this query to run in less than one second. If all the data is in memory, then will faster CPU and memory be the things that help? We have an alternate (a bit more conventional) server configuration that we are considering: HP ProLiant DL360p Gen 8 Dual Intel Xeon 3.3GHz 4-core E5-2643 CPUs 128GB PC3-12800 RAM 16x146GB 15K SAS hard drives HP Smart Array P822/2GB FBWC controller + P420i w/ 2GB FBWC + the usual accessories (optical drive, rail kit, dual power supplies) All suggestions welcomed! -Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} The responses: {noformat} From: Mike McCann Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 4:58:17 PM PDT To: Jeff Janes Cc: "pgsql-performance@postgresql.org" , Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden On May 13, 2013, at 4:24 PM, Jeff Janes wrote: > On Mon, May 13, 2013 at 3:36 PM, Mike McCann wrote: > > Increasing work_mem to 355 MB improves the performance by a factor of 2: > > stoqs_march2013_s=# set work_mem='355MB'; > SET > stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > > I tried changing random_page_cost to from 4 to 1 and saw no change. > > I'm wondering now what changes might get this query to run in less than one second. I think you are worrying about the wrong thing here. What is a web app going to do with 3,381,814 rows, once it obtains them? Your current testing is not testing the time it takes to stream that data to the client, or for the client to do something meaningful with that data. If you only plan to actually fetch a few dozen of those rows, then you probably need to incorporate that into your test, either by using a LIMIT, or by using a mock-up of the actual application to do some timings. Also, what is the type and collation of the column you are sorting on? non-'C' collations of text columns sort about 3 times slower than 'C' collation does. If all the data is in memory, then will faster CPU and memory be the things that help? Yes, those would help (it is not clear to me which of the two would help more), but I think you need to rethink your design of sending the entire database table to the application server for each page-view. Cheers, Jeff Hi Jeff, The datavalue column is double precision: stoqs_march2013_s=# \d+ stoqs_measuredparameter Table "public.stoqs_measuredparameter" Column | Type | Modifiers | Storage | Description ----------------+------------------+----------------------------------------------------------------------+---------+------------- id | integer | not null default nextval('stoqs_measuredparameter_id_seq'::regclass) | plain | measurement_id | integer | not null | plain | parameter_id | integer | not null | plain | datavalue | double precision | not null | plain | Indexes: "stoqs_measuredparameter_pkey" PRIMARY KEY, btree (id) "stoqs_measuredparameter_measurement_id_parameter_id_key" UNIQUE CONSTRAINT, btree (measurement_id, parameter_id) "stoqs_measuredparameter_datavalue" btree (datavalue) "stoqs_measuredparameter_measurement_id" btree (measurement_id) "stoqs_measuredparameter_parameter_id" btree (parameter_id) Foreign-key constraints: "stoqs_measuredparameter_measurement_id_fkey" FOREIGN KEY (measurement_id) REFERENCES stoqs_measurement(id) DEFERRABLE INITIALLY DEFERRED "stoqs_measuredparameter_parameter_id_fkey" FOREIGN KEY (parameter_id) REFERENCES stoqs_parameter(id) DEFERRABLE INITIALLY DEFERRED Has OIDs: no Thanks for the suggestion and advice to examine the web app performance. We've actually taken quite a few steps to optimize how the web app works. The example query I provided is a simple worst-case one that we can use to help us decide on the proper hardware. An actual query performed by the web app is: stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_march2013_s-# stoqs_parameter.name AS parameter__name, stoqs_march2013_s-# stoqs_parameter.standard_name AS parameter__standard_name, stoqs_march2013_s-# stoqs_measurement.depth AS measurement__depth, stoqs_march2013_s-# stoqs_measurement.geom AS measurement__geom, stoqs_march2013_s-# stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_march2013_s-# stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_march2013_s-# stoqs_measuredparameter.datavalue AS datavalue, stoqs_march2013_s-# stoqs_parameter.units AS parameter__units stoqs_march2013_s-# FROM stoqs_parameter p1, stoqs_march2013_s-# stoqs_measuredparameter stoqs_march2013_s-# INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) stoqs_march2013_s-# INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) stoqs_march2013_s-# INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) stoqs_march2013_s-# INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) stoqs_march2013_s-# INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) stoqs_march2013_s-# INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id stoqs_march2013_s-# WHERE (p1.name = 'sea_water_sigma_t') stoqs_march2013_s-# AND (mp1.datavalue > 25.19) stoqs_march2013_s-# AND (mp1.datavalue < 26.01) stoqs_march2013_s-# AND (mp1.parameter_id = p1.id) stoqs_march2013_s-# AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' stoqs_march2013_s(# AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' stoqs_march2013_s(# AND stoqs_parameter.name IN ('fl700_uncorr') stoqs_march2013_s(# AND stoqs_measurement.depth >= -1.88 stoqs_march2013_s(# AND stoqs_platform.name IN ('dorado') stoqs_march2013_s(# AND stoqs_measurement.depth <= 83.57) stoqs_march2013_s-# ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=770.211..770.211 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=770.200..770.200 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=770.036..770.036 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=91.369..680.072 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=91.346..577.633 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=91.280..531.408 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=91.202..116.140 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.002..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=91.194..109.846 rows=20170 loop s=1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=90.794..90.794 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=62.769..62.769 rows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=27.412..2 7.412 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.008..0. 017 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.012..0.012 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.007..0.010 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.004..0.004 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.004..0.004 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.151..0.151 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.035..0.145 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.005..0.069 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.014..0.014 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.008..0.012 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 770.445 ms (42 rows) We assume that steps taken to improve the worst-case query scenario will also improve these kind of queries. If anything above pops out as needing better planning please let us know that too! Thanks, Mike -- Mike McCann Software Engineer Monterey Bay Aquarium Research Institute 7700 Sandholdt Road Moss Landing, CA 95039-9644 Voice: 831.775.1769 Fax: 831.775.1736 http://www.mbari.org {noformat} Another response: {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 13, 2013 6:09:58 PM PDT To: Mike McCann Cc: Jeff Janes , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden On Mon, May 13, 2013 at 5:58 PM, Mike McCann wrote: > We assume that steps taken to improve the worst-case query scenario will > also improve these kind of queries. If anything above pops out as needing > better planning please let us know that too! Bad assumption. If your real workload will be queries like the one here that takes 700 ms, but you'll be running 10,000 of them a second, you're tuning / hardware choices are going to be much different then if your query is going to be the previous 7 second one. Use realistic queries, not ones that are nothing like what your real ones will be. then use pgbench and its ability to run custom sql scripts to get a REAL idea how your hardware performs. Note that if you will run the slow query you posted like once a minute and roll it up or cache it then don't get too worried about it. Pay attention to the queries that will add up, in aggregate, to your greatest load. {noformat} Not a whole lot of opinions except that tuning a server requires diligent work and performance is something one must do some work to achieve, and cannot be had by simply throwing money at the problem. h4. 20 May 2013 A few more responses over the weekend: {noformat} From: Greg Smith Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 7:44:16 PM PDT To: Mike McCann Cc: pgsql-performance@postgresql.org, Yuri Levinsky , Scott Marlowe , Julien Cigar , Arjen van der Meijden , Jeff Janes On 5/13/13 6:36 PM, Mike McCann wrote: > stoqs_march2013_s=# explain analyze select * from > stoqs_measuredparameter order by datavalue; > > QUERY PLAN > ------------------------------------------------------------------------------------------------------------------------------------------ > Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual > time=2503.078..2937.130 rows=3381814 loops=1) > Sort Key: datavalue > Sort Method: quicksort Memory: 362509kB > -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 > rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) > Total runtime: 3094.601 ms > (5 rows) > I tried changing random_page_cost to from 4 to 1 and saw no change. Have you tried putting an index by datavalue on this table? Once you've done that, then changing random_page_cost will make using that index look less expensive. Sorting chews through a good bit of CPU time, and that's where all of your runtime is being spent at--once you increase work_mem up very high that is. > I'm wondering now what changes might get this query to run in less than > one second. If all the data is in memory, then will faster CPU and > memory be the things that help? You're trying to fix a fundamental design issue with hardware. That usually doesn't go well. Once you get a box big enough to hold the whole database in RAM, beyond that the differences between server systems are relatively small. -- Greg Smith 2ndQuadrant US greg@2ndQuadrant.com Baltimore, MD PostgreSQL Training, Services, and 24x7 Support www.2ndQuadrant.com {noformat} and {noformat} From: Scott Marlowe Subject: Re: [PERFORM] Hardware suggestions for maximum read performance Date: May 19, 2013 8:57:32 PM PDT To: Greg Smith Cc: Mike McCann , "pgsql-performance@postgresql.org" , Yuri Levinsky , Julien Cigar , Arjen van der Meijden , Jeff Janes On Sun, May 19, 2013 at 8:44 PM, Greg Smith wrote: >On 5/13/13 6:36 PM, Mike McCann wrote: >> >> stoqs_march2013_s=# explain analyze select * from >> stoqs_measuredparameter order by datavalue; >> >> QUERY PLAN >> >>------------------------------------------------------------------------------------------------------------------------------------------ >> Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual >> time=2503.078..2937.130 rows=3381814 loops=1) >> Sort Key: datavalue >> Sort Method: quicksort Memory: 362509kB >> -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 >> rows=3381814 width=20) (actual time=0.016..335.745 rows=3381814 loops=1) >> Total runtime: 3094.601 ms >> (5 rows) >> >> I tried changing random_page_cost to from 4 to 1 and saw no change. > > > Have you tried putting an index by datavalue on this table? Once you've > done that, then changing random_page_cost will make using that index look > less expensive. Sorting chews through a good bit of CPU time, and that's > where all of your runtime is being spent at--once you increase work_mem up > very high that is. This++ plus cluster on that index if you can. {noformat} h4. 2 July 2013 We made the decision made to purchase a server with a FusionIO card. Server is now set up on the network with name "kraken". Here is the [confluence page for kraken's configuration settings| PostgreSQL Server kraken setup]. Setting up ability to run pgbench-tools on all the various servers. This may be done per user account by first installing from git and following the instructions in README.rst; however, the scripts need to run commands requiring createdb and vacuum privileges, so it's best to run as user postgres. So, logging into pgtest as user postgres and getting things set up: {noformat} -bash-4.1$ pwd /var/lib/pgsql -bash-4.1$ mkdir dev -bash-4.1$ cd dev -bash-4.1$ git clone git://git.postgresql.org/git/pgbench-tools.git Initialized empty Git repository in /var/lib/pgsql/dev/pgbench-tools/.git/ remote: Counting objects: 108, done. remote: Compressing objects: 100% (106/106), done. remote: Total 108 (delta 54), reused 0 (delta 0) Receiving objects: 100% (108/108), 24.61 KiB, done. Resolving deltas: 100% (54/54), done. -bash-4.1$ createdb results -bash-4.1$ createdb pgbench -bash-4.1$ cd pgbench-tools/ -bash-4.1$ psql -f init/resultdb.sql -d results psql:init/resultdb.sql:1: ERROR: table "timing" does not exist CREATE TABLE CREATE INDEX psql:init/resultdb.sql:12: ERROR: table "tests" does not exist psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_test_seq" for serial column "tests.test" psql:init/resultdb.sql:29: NOTICE: CREATE TABLE will create implicit sequence "tests_set_seq" for serial column "tests.set" CREATE TABLE psql:init/resultdb.sql:31: ERROR: table "testset" does not exist psql:init/resultdb.sql:36: NOTICE: CREATE TABLE will create implicit sequence "testset_set_seq" for serial column "testset.set" CREATE TABLE INSERT 0 1 psql:init/resultdb.sql:40: ERROR: table "test_bgwriter" does not exist CREATE TABLE -bash-4.1$ psql -c "UPDATE testset SET info='Initial test set' WHERE set=1" -d results UPDATE 1 {noformat} I think those ERROR messages can be ignored because the following tests work. The gnuplot package is needed for producing graphs of the results: {noformat} sudo -c "yum install gnuplot" {noformat} Edit the config file with hard path to pgbench and reduced number of scale and client runs so that we have a quick execution to just test things: {noformat} -bash-4.1$ mv config config.130702 -bash-4.1$ cp config.130702 config -bash-4.1$ vi config -bash-4.1$ diff config.130702 config 8c8 < PGBENCHBIN=`which pgbench` --- > PGBENCHBIN=/usr/pgsql-9.1/bin/pgbench 42,44c42,47 < SCALES="1 10 100 1000" < SETCLIENTS="1 2 4 8 16 32" < SETTIMES=3 --- > SCALES="1" > SETCLIENTS="1" > SETTIMES=1 > #SCALES="1 10 100 1000" > #SETCLIENTS="1 2 4 8 16 32" > #SETTIMES=3 {noformat} Edit scripts to use "local socket" connection (removing "-h $TESTHOST") for the postgres user: {noformat} -bash-4.1$ diff runset.130702 runset 19c19 < TESTPSQL="psql -h $TESTHOST -U $TESTUSER -p $TESTPORT -d $TESTDB" --- > TESTPSQL="psql -U $TESTUSER -p $TESTPORT -d $TESTDB" 24c24 < $PGBENCHBIN -i -s $SCALE -h $TESTHOST -U $TESTUSER -p $TESTPORT $TESTDB --- > $PGBENCHBIN -i -s $SCALE -U $TESTUSER -p $TESTPORT $TESTDB -bash-4.1$ diff benchwarmer.130702 benchwarmer 84,85c84,85 < TESTPSQL="psql -h $TESTHOST -U $TESTUSER -p $TESTPORT -d $TESTDB" < RESULTPSQL="psql -h $RESULTHOST -U $RESULTUSER -p $RESULTPORT -d $RESULTDB" --- > TESTPSQL="psql -U $TESTUSER -p $TESTPORT -d $TESTDB" > RESULTPSQL="psql -U $RESULTUSER -p $RESULTPORT -d $RESULTDB" 158c158 < $PGBENCHBIN -f $BASEDIR/$TESTDIR/$SCRIPT -s $SCALE -l -n $TESTLEN -U $TESTUSER -h $TESTHOST -p $TESTPORT -c $CLIENTS $WORKERS $TESTDB > results.txt & --- > $PGBENCHBIN -f $BASEDIR/$TESTDIR/$SCRIPT -s $SCALE -l -n $TESTLEN -U $TESTUSER -p $TESTPORT -c $CLIENTS $WORKERS $TESTDB > results.txt & -bash-4.1$ diff webreport.130702 webreport 10c10 < RESULTPSQL="psql -h $RESULTHOST -U $RESULTUSER -p $RESULTPORT -d $RESULTDB" --- > RESULTPSQL="psql -U $RESULTUSER -p $RESULTPORT -d $RESULTDB" {noformat} And now a test run looks like: {noformat} -bash-4.1$ ./runset Removing old pgbench tables DROP TABLE VACUUM Creating new pgbench tables NOTICE: table "pgbench_branches" does not exist, skipping NOTICE: table "pgbench_tellers" does not exist, skipping NOTICE: table "pgbench_accounts" does not exist, skipping NOTICE: table "pgbench_history" does not exist, skipping creating tables... 10000 tuples done. 20000 tuples done. 30000 tuples done. 40000 tuples done. 50000 tuples done. 60000 tuples done. 70000 tuples done. 80000 tuples done. 90000 tuples done. 100000 tuples done. set primary key... NOTICE: ALTER TABLE / ADD PRIMARY KEY will create implicit index "pgbench_branches_pkey" for table "pgbench_branches" NOTICE: ALTER TABLE / ADD PRIMARY KEY will create implicit index "pgbench_tellers_pkey" for table "pgbench_tellers" NOTICE: ALTER TABLE / ADD PRIMARY KEY will create implicit index "pgbench_accounts_pkey" for table "pgbench_accounts" vacuum...done. Run set #1 of 1 with 1 clients scale=1 Running tests using: psql -U postgres -p 5432 -d pgbench Storing results using: psql -U postgres -p 5432 -d results Cleaning up database pgbench TRUNCATE TABLE VACUUM CHECKPOINT INSERT 0 1 This is test 3 Script select.sql executing for 1 concurrent users... UPDATE 1 transaction type: Custom query scaling factor: 1 query mode: simple number of clients: 1 number of threads: 1 duration: 60 s number of transactions actually processed: 547599 tps = 9126.622772 (including connections establishing) tps = 9127.071214 (excluding connections establishing) Worst latency results: 2018 2311 2370 3155 3476 Could not find/open font when opening font "arial", using internal non-scalable font Could not find/open font when opening font "arial", using internal non-scalable font Could not find/open font when opening font "arial", using internal non-scalable font Warning: empty x range [1:1], adjusting to [0.99:1.01] Warning: empty y range [9228:9228], adjusting to [9135.72:9320.28] Warning: empty y2 range [21:21], adjusting to [20.79:21.21] Could not find/open font when opening font "arial", using internal non-scalable font Warning: empty x range [1:1], adjusting to [0.99:1.01] Warning: empty y range [9228:9228], adjusting to [9135.72:9320.28] Could not find/open font when opening font "arial", using internal non-scalable font Warning: empty x range [1:1], adjusting to [0.99:1.01] Warning: empty y range [1:1], adjusting to [0.99:1.01] Warning: empty z range [9228:9228], adjusting to [9135.72:9320.28] {noformat} h4. 3 July 2013 Making the results visible on an internally accessible web page. Turn on httpd and make an alias to the results directory that is produced above: {noformat} sudo /bin/bash chkconfig httpd on /sbin/service httpd start {noformat} Confirm that http://pgtest.shore.mbari.org/ is visible. It wasn't because the firewall on pgtest was preventing it. A call to Pat Allen fixed this. Next, edit /etc/httpd/conf/httpd.conf to add the alias and make the postgres home dir world executable (don't do this on a production system!): {noformat} {noformat} ]]> 21365335 2818082 2785317 9404604 9371854 5407418 5374687 3178651 3113130 21398107 21365381 16449673 16416910 16449669 16416906 5407410 5374679 31457281 19202049 1179692 2019-01-28 13:30:19.553 2019-01-28 13:30:19.553 16449667 16416904 33619972 33587204 31588365 cd /mbari/Tempbox/mccann <103 elvis.shore.mbari.org /Tempbox/mccann> tar cvf 2012.268.07.tar 2012.268.07 2012.268.07/ 2012.268.07/SmartSamplerDataLog.log 2012.268.07/Gulper.log ... 2012.268.07/ctdDriver2.cfg 2012.268.07/gps.log 2012.268.07/m3dmgx1.log <105 elvis.shore.mbari.org /Tempbox/mccann> gzip 2012.268.07.tar {noformat} # Copy tarball to /usr/local/auv_ctd and run the auvportal which submits the data to SSDS: {noformat} <255 elvis.shore.mbari.org /Tempbox/mccann> cp 2012.268.07.tar.gz /usr/local/auv_ctd <257 elvis.shore.mbari.org /local/auv_ctd> ./portal AUVCTD2012268 2012.268.07.tar.gz Tue, 25 Sep 2012 09:27:43 PDT: processConnection() got message: AUVCTD2012268;2012.268.07.tar.gz dstDir.getName() = 2012268 Unpacking /usr/local/auv_ctd/2012.268.07.tar.gz Copying /usr/local/auv_ctd/2012268/2012.268.07/ctdDriver2.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/ctdDriver2.log Copying /usr/local/auv_ctd/2012268/2012.268.07/gps.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/gps.log Copying /usr/local/auv_ctd/2012268/2012.268.07/isuslog.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/isuslog.log Copying /usr/local/auv_ctd/2012268/2012.268.07/devices.cfg to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/devices.cfg ... DEBUG [Thread-0] (XmlBuilder.java:538) - appendPrimitiveType called with method name isValidRole on object of class moos.ssds.metadata.DataProducer DEBUG [Thread-0] (XmlBuilder.java:482) - appendElement called on class moos.ssds.metadata.DataProducer using method getClass DEBUG [Thread-0] (XmlBuilder.java:278) - Converting java.lang.Class(class moos.ssds.metadata.DataProducer) to XML Marshalled the AUV mission in /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 into XML Attempting to publish AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ published XML describing mission AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ If everything goes well in SSDS:Ingest/Ruminate an email message to auvctd@listserver.mbari.org will arrive shortly. Tue, 25 Sep 2012 09:34:29 PDT: Done receiving AUV mission data. {noformat} Note: With the move of the portal to from jeckel to elvis the mailing to auvctd@listserver.mbari.org has stopped working. This needs to be debugged and fixed. # Process the LOPC data, e.g.: {noformat} <109 elvis.shore.mbari.org /u/ssdsadmin> cd ~/dev/auv_ctd/src/py/auv/ <109 elvis.shore.mbari.org /u/ssdsadmin> /bin/bash <109 elvis.shore.mbari.org /u/ssdsadmin> export PATH="/usr/local/miniconda2/bin:$PATH" <109 elvis.shore.mbari.org /u/ssdsadmin> source activate auv_ctd (auv_ctd) bash-4.2$ processLOPC.pl 2013.224.02 (auv_ctd) bash-4.2$ exit {noformat} # Once the mission's instrument netcdf files have been created (check pismo:/opt/ssds/logs/ssds-updateBot.log for successful processing) the overall Dorado science data processing script can be edited with the new survey to be processed: {noformat} <102 elvis.shore.mbari.org /u/ssdsadmin> cd dev/auv_ctd/src/matlab/doradosdp/process/ <103 elvis.shore.mbari.org /doradosdp/process> vi reprocess_surveys.m {noformat} # Add a line for the new survey to the end of the reprocess_surveys.m file, commenting out the previous survey that was processed: {noformat} % CANON September 2012 - laps around ESP drifter - last one for this campaign %%AUVprocess_main('AUVCTD', 2012, ['258.00'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') % CANON September 2012 - laps around ESP drifter - last one for this campaign AUVprocess_main('AUVCTD', 2012, ['268.07'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') {noformat} # Execute the Matlab job with a shell script that properly sets the DISPLAY environment and executes reprocess_surveys.m with matlab: {noformat} <104 elvis.shore.mbari.org /doradosdp/process> RunReprocessJob {noformat} # Check for error messages. If everything works an email is sent to the notifyEmail setting in reprocess_surveys.m, usually auvctd@listserver.mbari.org. h2. Software checkout and repository code management The Software for both the AUV Portal and the Science data processing are located in the _auv_ctd_ module on the CVS server moonjelly.shore.mbari.org. To check the software to a system use your favorite CVS client, or on Unix use these commands: {noformat} cd cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). {noformat} Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): {noformat} {noformat} > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... {noformat} h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: {noformat} usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy {noformat} For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: {noformat} adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 {noformat} Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look something like this: {noformat} bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ {noformat} \\ On the portal computer the screen will display text something like this: {noformat} Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS {noformat} Note: Detailed command-line instructions for "portaling" AUVCTD mission data into SSDS are available in the README file in the project or in the operational location on jeckel.shore.mbari.org in /usr/local/auv_ctd/ directory. There is also the portal.log file there which logs the output of the deamon process running on jeckel that receives and processes the data from AUV uploads from the vehicle. The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: {noformat} > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> {noformat} So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: {noformat} Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds {noformat} +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables and an Ocean Data View spreadsheet file. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file). If one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files.]]> 31555597 31588368 cd /mbari/Tempbox/mccann <103 elvis.shore.mbari.org /Tempbox/mccann> tar cvf 2012.268.07.tar 2012.268.07 2012.268.07/ 2012.268.07/SmartSamplerDataLog.log 2012.268.07/Gulper.log ... 2012.268.07/ctdDriver2.cfg 2012.268.07/gps.log 2012.268.07/m3dmgx1.log <105 elvis.shore.mbari.org /Tempbox/mccann> gzip 2012.268.07.tar {noformat} # Copy tarball to /usr/local/auv_ctd and run the auvportal which submits the data to SSDS: {noformat} <255 elvis.shore.mbari.org /Tempbox/mccann> cp 2012.268.07.tar.gz /usr/local/auv_ctd <256 elvis.shore.mbari.org /Tempbox/mccann> cd /usr/local/auv_ctd <257 elvis.shore.mbari.org /local/auv_ctd> ./portal AUVCTD2012268 2012.268.07.tar.gz Tue, 25 Sep 2012 09:27:43 PDT: processConnection() got message: AUVCTD2012268;2012.268.07.tar.gz dstDir.getName() = 2012268 Unpacking /usr/local/auv_ctd/2012.268.07.tar.gz Copying /usr/local/auv_ctd/2012268/2012.268.07/ctdDriver2.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/ctdDriver2.log Copying /usr/local/auv_ctd/2012268/2012.268.07/gps.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/gps.log Copying /usr/local/auv_ctd/2012268/2012.268.07/isuslog.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/isuslog.log Copying /usr/local/auv_ctd/2012268/2012.268.07/devices.cfg to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/devices.cfg ... DEBUG [Thread-0] (XmlBuilder.java:538) - appendPrimitiveType called with method name isValidRole on object of class moos.ssds.metadata.DataProducer DEBUG [Thread-0] (XmlBuilder.java:482) - appendElement called on class moos.ssds.metadata.DataProducer using method getClass DEBUG [Thread-0] (XmlBuilder.java:278) - Converting java.lang.Class(class moos.ssds.metadata.DataProducer) to XML Marshalled the AUV mission in /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 into XML Attempting to publish AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ published XML describing mission AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ If everything goes well in SSDS:Ingest/Ruminate an email message to auvctd@listserver.mbari.org will arrive shortly. Tue, 25 Sep 2012 09:34:29 PDT: Done receiving AUV mission data. {noformat} Note: With the move of the portal to from jeckel to elvis the mailing to auvctd@listserver.mbari.org has stopped working. This needs to be debugged and fixed. # Process the LOPC data, e.g.: {noformat} <109 elvis.shore.mbari.org /u/ssdsadmin> cd ~/dev/auv_ctd/src/py/auv/ <109 elvis.shore.mbari.org /u/ssdsadmin> /bin/bash <109 elvis.shore.mbari.org /u/ssdsadmin> export PATH="/usr/local/miniconda2/bin:$PATH" <109 elvis.shore.mbari.org /u/ssdsadmin> source activate auv_ctd (auv_ctd) bash-4.2$ processLOPC.pl 2013.224.02 (auv_ctd) bash-4.2$ exit {noformat} # Once the mission's instrument netcdf files have been created (check pismo:/opt/ssds/logs/ssds-updateBot.log for successful processing) the overall Dorado science data processing script can be edited with the new survey to be processed: {noformat} <102 elvis.shore.mbari.org /u/ssdsadmin> cd dev/auv_ctd/src/matlab/doradosdp/process/ <103 elvis.shore.mbari.org /doradosdp/process> vi reprocess_surveys.m {noformat} # Add a line for the new survey to the end of the reprocess_surveys.m file, commenting out the previous survey that was processed: {noformat} % CANON September 2012 - laps around ESP drifter - last one for this campaign %%AUVprocess_main('AUVCTD', 2012, ['258.00'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') % CANON September 2012 - laps around ESP drifter - last one for this campaign AUVprocess_main('AUVCTD', 2012, ['268.07'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') {noformat} # Execute the Matlab job with a shell script that properly sets the DISPLAY environment and executes reprocess_surveys.m with matlab: {noformat} <104 elvis.shore.mbari.org /doradosdp/process> RunReprocessJob {noformat} # Check for error messages. If everything works an email is sent to the notifyEmail setting in reprocess_surveys.m, usually auvctd@listserver.mbari.org. h2. Software checkout and repository code management The Software for both the AUV Portal and the Science data processing are located in the _auv_ctd_ module on the CVS server moonjelly.shore.mbari.org. To check the software to a system use your favorite CVS client, or on Unix use these commands: {noformat} cd cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). {noformat} Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): {noformat} {noformat} > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... {noformat} h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: {noformat} usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy {noformat} For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: {noformat} adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 {noformat} Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look something like this: {noformat} bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ {noformat} \\ On the portal computer the screen will display text something like this: {noformat} Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS {noformat} Note: Detailed command-line instructions for "portaling" AUVCTD mission data into SSDS are available in the README file in the project or in the operational location on jeckel.shore.mbari.org in /usr/local/auv_ctd/ directory. There is also the portal.log file there which logs the output of the deamon process running on jeckel that receives and processes the data from AUV uploads from the vehicle. The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: {noformat} > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> {noformat} So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: {noformat} Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds {noformat} +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables and an Ocean Data View spreadsheet file. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file). If one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files.]]> 31555600 31588367 cd /mbari/Tempbox/mccann <103 elvis.shore.mbari.org /Tempbox/mccann> tar cvf 2012.268.07.tar 2012.268.07 2012.268.07/ 2012.268.07/SmartSamplerDataLog.log 2012.268.07/Gulper.log ... 2012.268.07/ctdDriver2.cfg 2012.268.07/gps.log 2012.268.07/m3dmgx1.log <105 elvis.shore.mbari.org /Tempbox/mccann> gzip 2012.268.07.tar {noformat} # Copy tarball to /usr/local/auv_ctd and run the auvportal which submits the data to SSDS: {noformat} <255 elvis.shore.mbari.org /Tempbox/mccann> cp 2012.268.07.tar.gz /usr/local/auv_ctd <257 elvis.shore.mbari.org /local/auv_ctd> ./portal AUVCTD2012268 2012.268.07.tar.gz Tue, 25 Sep 2012 09:27:43 PDT: processConnection() got message: AUVCTD2012268;2012.268.07.tar.gz dstDir.getName() = 2012268 Unpacking /usr/local/auv_ctd/2012.268.07.tar.gz Copying /usr/local/auv_ctd/2012268/2012.268.07/ctdDriver2.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/ctdDriver2.log Copying /usr/local/auv_ctd/2012268/2012.268.07/gps.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/gps.log Copying /usr/local/auv_ctd/2012268/2012.268.07/isuslog.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/isuslog.log Copying /usr/local/auv_ctd/2012268/2012.268.07/devices.cfg to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/devices.cfg ... DEBUG [Thread-0] (XmlBuilder.java:538) - appendPrimitiveType called with method name isValidRole on object of class moos.ssds.metadata.DataProducer DEBUG [Thread-0] (XmlBuilder.java:482) - appendElement called on class moos.ssds.metadata.DataProducer using method getClass DEBUG [Thread-0] (XmlBuilder.java:278) - Converting java.lang.Class(class moos.ssds.metadata.DataProducer) to XML Marshalled the AUV mission in /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 into XML Attempting to publish AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ published XML describing mission AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ If everything goes well in SSDS:Ingest/Ruminate an email message to auvctd@listserver.mbari.org will arrive shortly. Tue, 25 Sep 2012 09:34:29 PDT: Done receiving AUV mission data. {noformat} Note: With the move of the portal to from jeckel to elvis the mailing to auvctd@listserver.mbari.org has stopped working. This needs to be debugged and fixed. # Process the LOPC data, e.g.: {noformat} <109 elvis.shore.mbari.org /u/ssdsadmin> cd ~/dev/auv_ctd/src/py/auv/ <109 elvis.shore.mbari.org /u/ssdsadmin> /bin/bash <109 elvis.shore.mbari.org /u/ssdsadmin> export PATH="/usr/local/miniconda2/bin:$PATH" <109 elvis.shore.mbari.org /u/ssdsadmin> source activate auv_ctd (auv_ctd) bash-4.2$ processLOPC.pl 2013.224.02 (auv_ctd) bash-4.2$ exit {noformat} # Once the mission's instrument netcdf files have been created (check pismo:/opt/ssds/logs/ssds-updateBot.log for successful processing) the overall Dorado science data processing script can be edited with the new survey to be processed: {noformat} <102 elvis.shore.mbari.org /u/ssdsadmin> cd dev/auv_ctd/src/matlab/doradosdp/process/ <103 elvis.shore.mbari.org /doradosdp/process> vi reprocess_surveys.m {noformat} # Add a line for the new survey to the end of the reprocess_surveys.m file, commenting out the previous survey that was processed: {noformat} % CANON September 2012 - laps around ESP drifter - last one for this campaign %%AUVprocess_main('AUVCTD', 2012, ['258.00'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') % CANON September 2012 - laps around ESP drifter - last one for this campaign AUVprocess_main('AUVCTD', 2012, ['268.07'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') {noformat} # Execute the Matlab job with a shell script that properly sets the DISPLAY environment and executes reprocess_surveys.m with matlab: {noformat} <104 elvis.shore.mbari.org /doradosdp/process> RunReprocessJob {noformat} # Check for error messages. If everything works an email is sent to the notifyEmail setting in reprocess_surveys.m, usually auvctd@listserver.mbari.org. h2. Software checkout and repository code management The Software for both the AUV Portal and the Science data processing are located in the _auv_ctd_ module on the CVS server moonjelly.shore.mbari.org. To check the software to a system use your favorite CVS client, or on Unix use these commands: {noformat} cd cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). {noformat} Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): {noformat} {noformat} > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... {noformat} h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: {noformat} usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy {noformat} For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: {noformat} adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 {noformat} Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look something like this: {noformat} bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ {noformat} \\ On the portal computer the screen will display text something like this: {noformat} Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS {noformat} Note: Detailed command-line instructions for "portaling" AUVCTD mission data into SSDS are available in the README file in the project or in the operational location on jeckel.shore.mbari.org in /usr/local/auv_ctd/ directory. There is also the portal.log file there which logs the output of the deamon process running on jeckel that receives and processes the data from AUV uploads from the vehicle. The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: {noformat} > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> {noformat} So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: {noformat} Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds {noformat} +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables and an Ocean Data View spreadsheet file. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file). If one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files.]]> 31555599 22544410 cd /mbari/Tempbox/mccann <103 elvis.shore.mbari.org /Tempbox/mccann> tar cvf 2012.268.07.tar 2012.268.07 2012.268.07/ 2012.268.07/SmartSamplerDataLog.log 2012.268.07/Gulper.log ... 2012.268.07/ctdDriver2.cfg 2012.268.07/gps.log 2012.268.07/m3dmgx1.log <105 elvis.shore.mbari.org /Tempbox/mccann> gzip 2012.268.07.tar {noformat} # Copy tarball to /usr/local/auv_ctd and run the auvportal which submits the data to SSDS: {noformat} <255 elvis.shore.mbari.org /Tempbox/mccann> cp 2012.268.07.tar.gz /usr/local/auv_ctd <257 elvis.shore.mbari.org /local/auv_ctd> ./portal AUVCTD2012268 2012.268.07.tar.gz Tue, 25 Sep 2012 09:27:43 PDT: processConnection() got message: AUVCTD2012268;2012.268.07.tar.gz dstDir.getName() = 2012268 Unpacking /usr/local/auv_ctd/2012.268.07.tar.gz Copying /usr/local/auv_ctd/2012268/2012.268.07/ctdDriver2.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/ctdDriver2.log Copying /usr/local/auv_ctd/2012268/2012.268.07/gps.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/gps.log Copying /usr/local/auv_ctd/2012268/2012.268.07/isuslog.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/isuslog.log Copying /usr/local/auv_ctd/2012268/2012.268.07/devices.cfg to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/devices.cfg ... DEBUG [Thread-0] (XmlBuilder.java:538) - appendPrimitiveType called with method name isValidRole on object of class moos.ssds.metadata.DataProducer DEBUG [Thread-0] (XmlBuilder.java:482) - appendElement called on class moos.ssds.metadata.DataProducer using method getClass DEBUG [Thread-0] (XmlBuilder.java:278) - Converting java.lang.Class(class moos.ssds.metadata.DataProducer) to XML Marshalled the AUV mission in /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 into XML Attempting to publish AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ published XML describing mission AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ If everything goes well in SSDS:Ingest/Ruminate an email message to auvctd@listserver.mbari.org will arrive shortly. Tue, 25 Sep 2012 09:34:29 PDT: Done receiving AUV mission data. {noformat} Note: With the move of the portal to from jeckel to elvis the mailing to auvctd@listserver.mbari.org has stopped working. This needs to be debugged and fixed. # Process the LOPC data, e.g.: {noformat} <109 elvis.shore.mbari.org /u/ssdsadmin> cd ~/dev/auv_ctd/src/py/auv/ <110 elvis.shore.mbari.org /py/auv> processLOPC.pl 2013.224.02 {noformat} # Once the mission's instrument netcdf files have been created (check pismo:/opt/ssds/logs/ssds-updateBot.log for successful processing) the overall Dorado science data processing script can be edited with the new survey to be processed: {noformat} <102 elvis.shore.mbari.org /u/ssdsadmin> cd dev/auv_ctd/src/matlab/doradosdp/process/ <103 elvis.shore.mbari.org /doradosdp/process> vi reprocess_surveys.m {noformat} # Add a line for the new survey to the end of the reprocess_surveys.m file, commenting out the previous survey that was processed: {noformat} % CANON September 2012 - laps around ESP drifter - last one for this campaign %%AUVprocess_main('AUVCTD', 2012, ['258.00'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') % CANON September 2012 - laps around ESP drifter - last one for this campaign AUVprocess_main('AUVCTD', 2012, ['268.07'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') {noformat} # Execute the Matlab job with a shell script that properly sets the DISPLAY environment and executes reprocess_surveys.m with matlab: {noformat} <104 elvis.shore.mbari.org /doradosdp/process> RunReprocessJob {noformat} # Check for error messages. If everything works an email is sent to the notifyEmail setting in reprocess_surveys.m, usually auvctd@listserver.mbari.org. h2. Software checkout and repository code management The Software for both the AUV Portal and the Science data processing are located in the _auv_ctd_ module on the CVS server moonjelly.shore.mbari.org. To check the software to a system use your favorite CVS client, or on Unix use these commands: {noformat} cd cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). {noformat} Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): {noformat} {noformat} > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... {noformat} h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: {noformat} usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy {noformat} For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: {noformat} adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 {noformat} Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look something like this: {noformat} bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ {noformat} \\ On the portal computer the screen will display text something like this: {noformat} Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS {noformat} Note: Detailed command-line instructions for "portaling" AUVCTD mission data into SSDS are available in the README file in the project or in the operational location on jeckel.shore.mbari.org in /usr/local/auv_ctd/ directory. There is also the portal.log file there which logs the output of the deamon process running on jeckel that receives and processes the data from AUV uploads from the vehicle. The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: {noformat} > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> {noformat} So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: {noformat} Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds {noformat} +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables and an Ocean Data View spreadsheet file. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file). If one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files.]]> 22511642 31588357 31555589 26673717 26640961 5865499 5832735 5865500 5832736 21398119 21365395 5865498 5832734 19074290 19041609 14549009 14516242 14549010 14516243 9830777 9798021 19074288 19041607 19074287 19041606 19074278 19041597 16449632 16416869 9404448 9371684 10387668 10354904 9404447 9371683 9404449 9371685 19074298 19041617 19074295 19041614 19074296 19041615 19074293 19041612 14549014 14516247 10387680 10354915 3178575 3113043 21004837 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. ]]> 20972074 10387679 10354914 19074291 19041610 14549012 14516245 10387682 10354917 21398210 # Allow connections from mbari hosts inside our firewall - mpm 7 May 2013 > listen_addresses = '*' # what IP address(es) to listen on; 64c65,66 < max_connections = 100 # (change requires restart) --- > # Changed max_connections from 100 to 10 corresponding to increase in work_mem - mpm 22 May 2013 > max_connections = 10 # (change requires restart) 109c111 < shared_buffers = 32MB # min 128kB --- > shared_buffers = 3GB # min 128kB 118c120,122 < #work_mem = 1MB # min 64kB --- > > # Changed from 1 MB to 400MB to support large STOQS sorts - mpm 22 May 2013 > work_mem = 400MB # min 64kB 174c178,179 < #checkpoint_segments = 3 # in logfile segments, min 1, 16MB each --- > # pgbench-tools indicated high checkpoint frequency, adjust with guidance from p.137. dr 3 July 2013 > checkpoint_segments = 10 # in logfile segments, min 1, 16MB each 176c181 < #checkpoint_completion_target = 0.5 # checkpoint target duration, 0.0 - 1.0 --- > checkpoint_completion_target = 0.9 # checkpoint target duration, 0.0 - 1.0 244c249,251 < #random_page_cost = 4.0 # same scale as above --- > > # Changed to 1.01 per Greg Smith's suggestion on postgresql-perform list - seems to improve STOQS mp query performance - mpm 14 May 2013 > random_page_cost = 1.01 # same scale as above 359c366,367 < #log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements --- > # Log all queries taking longer than 1 second. mpm 15 July 2013 > log_min_duration_statement = 1000 # -1 is disabled, 0 logs all statements 377c385,387 < #log_line_prefix = '' # special values: --- > # Set timestamp for log files to be compatible with pgFouine. mpm 15 July 2013 > log_line_prefix = '%t [%p]: [%l-1] user=%u, db=%d' > # special values: 399c409,410 < #log_temp_files = -1 # log temporary files equal or larger --- > # Turned on logging of temp files. See $PGDATA/base//pgsql_tmp for what a DB is doing - mpm 15 May 2013 > log_temp_files = 0 # log temporary files equal or larger 482c493,494 < #timezone = '(defaults to server environment setting)' --- > # Severs running STOQS databases must have timezone set to 'GMT' > timezone = 'GMT' {noformat} pgbench-tools results at: http://kraken.shore.mbari.org/pgbench-tools/results.130716.AfterPostgresql.confChanges_tests_133_204/index.htm (as expected, no appreciable changes in performance)]]> 21365488 10387684 10354919 21398204 diff sysctl.conf.130712 sysctl.conf 40a41,53 > > # Swappiness set to 0 per p.81 in PostgreSQL 9.0 High Performance book. mpm - 12 July 2013 > vm.swappiness = 0 > # Reduce tendency to allocate more ram than the system has. mpm - 12 July 2013 > vm.overcommit_memory = 2 > > # Setting for shared memory = 1/2 physical RAM = 64 GB. mpm 12 July 2013 > ##kernel.shmmax = 67649847296 > ##kernel.shmall = 16516076 > > # Set shared memory to 4 GB - a better value for PostgreSQL on large memory systems. mpm 12 July 2013 > kernel.shmmax = 4228112384 > kernel.shmall = 1032254 <198 kraken.shore.mbari.org //etc> diff fstab.130712 fstab 11,13c11,13 < /dev/mapper/vg_kraken-LogVol02 /pg_xlog_spinning ext4 defaults 1 2 < /dev/mapper/vg_kraken-LogVol04 /pgdata_spinning ext4 defaults 1 2 < /dev/mapper/vg_kraken-LogVol03 /tablespace_spinning ext4 defaults 1 2 --- > /dev/mapper/vg_kraken-LogVol02 /pg_xlog_spinning ext4 defaults,noatime 1 2 > /dev/mapper/vg_kraken-LogVol04 /pgdata_spinning ext4 defaults,noatime 1 2 > /dev/mapper/vg_kraken-LogVol03 /tablespace_spinning ext4 defaults,noatime 1 2 22,24c22,24 < /dev/mapper/iomemory_vg-iomemVol1 /pg_xlog ext4 defaults,noauto,discard 0 0 < /dev/mapper/iomemory_vg-iomemVol2 /tablespace ext4 defaults,noauto,discard 0 0 < /dev/mapper/iomemory_vg-iomemVol3 /pgdata ext4 defaults,noauto,discard 0 0 --- > /dev/mapper/iomemory_vg-iomemVol1 /pg_xlog ext4 defaults,noatime,noauto,discard 0 0 > /dev/mapper/iomemory_vg-iomemVol2 /tablespace ext4 defaults,noatime,noauto,discard 0 0 > /dev/mapper/iomemory_vg-iomemVol3 /pgdata ext4 defaults,noatime,noauto,discard 0 0 <199 kraken.shore.mbari.org //etc> diff rc.local.130712 rc.local 7a8,9 > blockdev --setra 8192 /dev/mapper/vg_kraken-LogVol04 > blockdev --setra 8192 /dev/mapper/iomemory_vg-iomemVol3 {noformat} pgbench-tools results at: http://kraken.shore.mbari.org/pgbench-tools/results.130715.AfterOSConfigChanges_tests_61_132/index.htm ]]> 21365482 21004823 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h 4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. ]]> 20972060 9404455 9371691 21004825 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. ]]> 20972062 9404453 9371689 9404454 9371690 9404451 9371687 21004822 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} ]]> 20972059 9830735 9797969 21004805 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. ]]> 20972042 10944798 10912039 10944796 10912037 16449567 16416800 9830716 9797949 16449565 16416798 9830703 9797936 21398267 21365547 5407259 5374524 10387623 10354859 9830690 9797922 10387641 10354877 10944813 10912054 8913275 8946028 1 2007-06-08 14:58:06.287 2007-06-08 14:58:06.287 1179692 21398246 ;date.timezone = {noformat} Run a pgbench test to generate some query traffic. {noformat} -bash-4.1$ ./pgbench -T 60 -c 4 pgbench starting vacuum...end. transaction type: TPC-B (sort of) scaling factor: 160 query mode: simple number of clients: 4 number of threads: 1 duration: 60 s number of transactions actually processed: 34324 tps = 571.932873 (including connections establishing) tps = 572.082084 (excluding connections establishing) {noformat} Run pgfouine on todays log file to make sure it works. *note: The command starts with pgfouine.php not just pgfouine. {noformat} -bash-4.1$ pgfouine.php -logtype stderr -file /var/lib/pgsql/9.1/data/pg_log/postgresql-Wed.log ##### Overall statistics ##### Number of unique normalized queries: 1 Number of queries: 1 Total query duration: 0.0s ##### Queries by type ##### SELECT: 1 100.0% ##### Queries that took up the most time (N) ##### 1) 0.0s - 1 - select count(*) from pgbench_branches -- ##### Slowest queries ##### 1) 0.00 s - select count(*) from pgbench_branches -- ##### Most frequent queries (N) ##### 1) 1 - 0.0s - select count(*) from pgbench_branches -- ##### Slowest queries (N) ##### 1) 0.00 - 1 - select count(*) from pgbench_branches -- {noformat} ]]> 21365524 21398245 21365523 10944805 10912046 10387643 10354879 29229304 29196536 21433961 21365482 2013-07-16 12:04:09.073 2013-07-16 12:04:09.073 29229306 29196538 29229305 29196537 9404824 9372078 29229300 29196532 29229302 29196534 9404819 9372073 9404820 9372074 9404833 9372087 10388770 10356044 10388768 10356042 29229308 29196540 21398284 diff /etc/php.ini.130723 /etc/php.ini 946c946 < ;date.timezone = --- > date.timezone = America/Los_Angeles {noformat} Create a web directory for pgFouine reports and test running it: {noformat} sudo mkdir /var/www/html/pgfouine sudo chown postgres /var/www/html/pgfouine pgfouine.php -logtype stderr -file /var/lib/pgsql/9.1/data/pg_log/postgresql-Mon.log -format html > /var/www/html/pgfouine/InitialTest.html {noformat} Results viewable at http://kraken.shore.mbari.org/pgfouine/InitialTest.html ]]> 21365564 10387802 10355062 9404827 9372081 9404828 9372082 9404810 9372064 9404808 9372062 9404806 9372060 9404805 9372059 21398303 log_filename = 'postgresql-%Y%m%d.log' # log file name pattern, 367c367 < log_min_duration_statement = 100 # -1 is disabled, 0 logs all statements --- > log_min_duration_statement = 10 # -1 is disabled, 0 logs all statements {noformat} pgFouine now: {noformat} pgfouine.php -logtype stderr -file /var/lib/pgsql/9.1/data/pg_log/postgresql-20130723.log -format html > /var/www/html/pgfouine/postgresql-20130723.html {noformat} gives: http://kraken.shore.mbari.org/pgfouine/postgresql-20130723.html The pgfouine.php command is something we can put into cron, or just run it when we need to and examine the reports in http://kraken.shore.mbari.org/pgfouine/. ]]> 21365583 9404817 9372071 9404815 9372069 21398302 log_filename = 'postgresql-%Y%m%d.log' # log file name pattern, {noformat} Example of old file name format compared to new format. Old format: {noformat} postgresql-Fri.log {noformat} New format: {noformat} postgresql-20130723.log {noformat}]]> 21365582 9404811 9372065 9404855 9372110 9404856 9372111 9404858 9372113 9404853 9372108 9830598 9797830 9830597 9797829 9404860 9372115 10387834 10355097 9404862 9372117 10387819 10355081 10387822 10355084 9404842 9372096 10388772 10356046 9404838 9372092 10388773 10356047 9404848 9372102 9404844 9372098 10387815 10355077 9404846 9372100 10387817 10355079 9404883 9372140 9404884 9372141 9830576 9797808 9404887 9372145 9830563 9797794 19202049 2012-03-19 15:39:14.567 2019-01-28 13:31:54.337 9830590 9797822 9830592 9797824 9404873 9372129 9404872 9372128 9404877 9372134 9404875 9372132 9404881 9372138 9830585 9797817 9404916 9372175 9404918 9372177 9404920 9372179 9404923 9372182 9830534 9797765 9830533 9797764 9404925 9372184 9404926 9372185 9830531 9797762 9404928 9372187 10387764 10355011 9404901 9372160 9404903 9372162 9404907 9372166 9404909 9372168 12025859 12058625 1179692 2010-04-28 15:37:00.823 2010-04-28 15:37:00.823 9404911 9372170 9404913 9372172 19074320 19041640 29523970 2 1179692 2016-11-14 10:42:24.593 2016-11-14 10:42:24.593 9830527 9797758 10387922 10355187 9831437 cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look somehting like this: bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ \\ On the portal computer the screen will display text something like this: Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS Once the AUV files have been submitted to SSDS they are available via SSDS's Hierarchical Ocean Observatory Visualization and Editing System (HOOVES) program. The data from these files may be plotted and retrieved from HOOVES. (Note that the MultiGenerateNetcdfTask that creates the .nc files takes several minutes and that you may need to wait up to an hour before querying SSDS with HOOVES to see the netCDF data files.) | | | *Figure 2*. HOOVES showing AUV Hydroscat instrument data processing after submission of data to SSDS. Clicking on the hydroscatlog.nc button brings up a pane where variables may be selected for plotting and numeric display. | The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey | | | *Figure 3*. HOOVES display of processing tree graph for level 1 and level 2 data products. (Click on image for full processing tree graph.) | To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables (which can be plotted against the original instrument netCDF files as a time series in HOOVES) and an Ocean Data View spreadsheet file. The latter may be loaded into ODV directly from HOOVES. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file), therefore HOOVES supports the registration of the creation of a Collection by an ODV user with permission to write to the \\tornado\AUVCTD\surveys\\odv directory. Clicking on the Level-2 .txt file button will import the survey data into HOOVES. If the user then saves that collection along with section splines back to the surveys\\odv directory then thie "ProcessRun" should be entered into SSDS. This is easily done by the user entering their email address on this form and clicking the Submit ProcessRun button: | | | *Figure 4*. Submitting a ProcessRun from HOOVES that registers the existence of an ODV Collection in SSDS. | If an ODV collection is available then that may be used - saving time in selecting the zoom, coastline layer, and section spline. An example screen shot of launching an ODV Collection from HOOVES is shown below. The operation of ODV is beyond the scope of this document. However, if one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files. | | | *Figure 5*. Launching Ocean Data View from HOOVES for visualization with and interaction with AUV survey data. (Click on image for larger view.) | |]]> 9798698 9404735 9371986 9404738 9371989 9404737 9371988 9404733 9371984 11829403 cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). {noformat} Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): {noformat} {noformat} > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... {noformat} h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: {noformat} usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy {noformat} For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: {noformat} adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 {noformat} Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look something like this: {noformat} bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ {noformat} \\ On the portal computer the screen will display text something like this: {noformat} Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS {noformat} Once the AUV files have been submitted to SSDS they are available via SSDS's Hierarchical Ocean Observatory Visualization and Editing System (HOOVES) program. The data from these files may be plotted and retrieved from HOOVES. (Note that the MultiGenerateNetcdfTask that creates the .nc files takes several minutes and that you may need to wait up to an hour before querying SSDS with HOOVES to see the netCDF data files.) | | | *Figure 2*. HOOVES showing AUV Hydroscat instrument data processing after submission of data to SSDS. Clicking on the hydroscatlog.nc button brings up a pane where variables may be selected for plotting and numeric display. | The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey | | | *Figure 3*. HOOVES display of processing tree graph for level 1 and level 2 data products. (Click on image for full processing tree graph.) | To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: {noformat} > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> {noformat} So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: {noformat} Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds {noformat} +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables (which can be plotted against the original instrument netCDF files as a time series in HOOVES) and an Ocean Data View spreadsheet file. The latter may be loaded into ODV directly from HOOVES. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file), therefore HOOVES supports the registration of the creation of a Collection by an ODV user with permission to write to the \\tornado\AUVCTD\surveys\\odv directory. Clicking on the Level-2 .txt file button will import the survey data into HOOVES. If the user then saves that collection along with section splines back to the surveys\\odv directory then thie "ProcessRun" should be entered into SSDS. This is easily done by the user entering their email address on this form and clicking the Submit ProcessRun button: | | | *Figure 4*. Submitting a ProcessRun from HOOVES that registers the existence of an ODV Collection in SSDS. | If an ODV collection is available then that may be used - saving time in selecting the zoom, coastline layer, and section spline. An example screen shot of launching an ODV Collection from HOOVES is shown below. The operation of ODV is beyond the scope of this document. However, if one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files. | | | *Figure 5*. Launching Ocean Data View from HOOVES for visualization with and interaction with AUV survey data. (Click on image for larger view.) | ]]> 11796644 11829405 cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). {noformat} Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): {noformat} {noformat} > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... {noformat} h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: {noformat} usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy {noformat} For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: {noformat} adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 {noformat} Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look something like this: {noformat} bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ {noformat} \\ On the portal computer the screen will display text something like this: {noformat} Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS {noformat} Note: Detailed command-line instructions for "portaling" AUVCTD mission data into SSDS are available in the README file in the project or in the operational location on jeckel.shore.mbari.org in /usr/local/auv_ctd/ directory. There is also the portal.log file there which logs the output of the deamon process running on jeckel that receives and processes the data from AUV uploads from the vehicle. The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey | | | *Figure 3*. HOOVES display of processing tree graph for level 1 and level 2 data products. (Click on image for full processing tree graph.) | To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: {noformat} > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> {noformat} So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: {noformat} Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds {noformat} +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables (which can be plotted against the original instrument netCDF files as a time series in HOOVES) and an Ocean Data View spreadsheet file. The latter may be loaded into ODV directly from HOOVES. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file), therefore HOOVES supports the registration of the creation of a Collection by an ODV user with permission to write to the \\tornado\AUVCTD\surveys\\odv directory. Clicking on the Level-2 .txt file button will import the survey data into HOOVES. If the user then saves that collection along with section splines back to the surveys\\odv directory then thie "ProcessRun" should be entered into SSDS. This is easily done by the user entering their email address on this form and clicking the Submit ProcessRun button: | | | *Figure 4*. Submitting a ProcessRun from HOOVES that registers the existence of an ODV Collection in SSDS. | If an ODV collection is available then that may be used - saving time in selecting the zoom, coastline layer, and section spline. An example screen shot of launching an ODV Collection from HOOVES is shown below. The operation of ODV is beyond the scope of this document. However, if one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files. | | | *Figure 5*. Launching Ocean Data View from HOOVES for visualization with and interaction with AUV survey data. (Click on image for larger view.) | ]]> 11796646 11829407 cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). {noformat} Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): {noformat} {noformat} > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... {noformat} h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: {noformat} usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy {noformat} For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: {noformat} adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 {noformat} Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look something like this: {noformat} bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ {noformat} \\ On the portal computer the screen will display text something like this: {noformat} Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS {noformat} Note: Detailed command-line instructions for "portaling" AUVCTD mission data into SSDS are available in the README file in the project or in the operational location on jeckel.shore.mbari.org in /usr/local/auv_ctd/ directory. There is also the portal.log file there which logs the output of the deamon process running on jeckel that receives and processes the data from AUV uploads from the vehicle. The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: {noformat} > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> {noformat} So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: {noformat} Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds {noformat} +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables (which can be plotted against the original instrument netCDF files as a time series in HOOVES) and an Ocean Data View spreadsheet file. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file). If one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files. ]]> 11796648 9404723 Dives can be viewed in 3D replay the morning after editing or re-editing, unless the dive occurred more than 30 days ago. To view newly-edited dives older than a month old, contact Mike McCann and he will reload them.]]> 9371974 9404726 \*To view newly-edited or re-edited dives older than a month old, contact Mike McCann and he will reload them.   Search for the desired dive in Expedition Database.  When the expedition is displayed, choose "3D Replay" The Construct 3D Replay window will appear.  The 3D replay program can only be run with Netscape so copy and paste the url from the 'Construct 3D Replay' page into Netscape and click 'Generate 3D Replay' Further instructions for navigating with 3D replay can be found at http://mww.mbari.org/3Dreplay/geoVRMLreqts.html\\]]> 9371977 9404725 9371976 9404719 9371970 9404720 9371971 30736642 30703906 30736640 30703904 9404717 9371968 20119843 20087082 20119844 20087083 9404714 9371965 21397506 /pgdata/data -rw------- 1 postgres postgres 1749 May 6 16:03 pgstartup.log [root@pgtest 9.1]# cd data [root@pgtest data]# ls -l total 84 drwx------ 6 postgres postgres 4096 May 6 16:05 base drwx------ 2 postgres postgres 4096 May 6 16:12 global drwx------ 2 postgres postgres 4096 May 6 15:29 pg_clog -rw------- 1 postgres postgres 4232 May 6 15:29 pg_hba.conf -rw------- 1 postgres postgres 1636 May 6 15:29 pg_ident.conf drwx------ 2 postgres postgres 4096 May 6 15:33 pg_log drwx------ 4 postgres postgres 4096 May 6 15:29 pg_multixact drwx------ 2 postgres postgres 4096 May 6 16:03 pg_notify drwx------ 2 postgres postgres 4096 May 6 15:29 pg_serial drwx------ 2 postgres postgres 4096 May 6 16:30 pg_stat_tmp drwx------ 2 postgres postgres 4096 May 6 15:29 pg_subtrans lrwxrwxrwx 1 root root 21 May 6 15:57 pg_tblspc -> /tablespace/pg_tblspc drwx------ 2 postgres postgres 4096 May 6 15:29 pg_twophase -rw------- 1 postgres postgres 4 May 6 15:29 PG_VERSION lrwxrwxrwx 1 root root 16 May 6 15:38 pg_xlog -> /pg_xlog/pg_xlog -rw------- 1 postgres postgres 19146 May 6 15:29 postgresql.conf -rw------- 1 postgres postgres 71 May 6 16:03 postmaster.opts -rw------- 1 postgres postgres 81 May 6 16:03 postmaster.pid {noformat} Starting on changing default system configurations (previous .conf file moved to .YYMMDD file, copied to and edited): # /etc/sysctl.conf: vm.swappiness = 0 and vm.overcommit_memory = 2 # /etc/fstab: Set noatime on the database partitions # /etc/rc.local: blockdev --setra 8192 /dev/sdb1 (pgdata) h4. 7 May 2013 Postgresql & system firewall configuration changes to allow connections across the network (see diffs from .130507 conf files): # Edited /var/lib/pgsql/9.1/data/pg_hba.conf # Edited /var/lib/pgsql/9.1/data/postgresql.conf # Executed "sudo python /usr/bin/system-config-firewall" and added 5432:tcp & 5432:udp From machine odss-test: # Added stoqs_september2012_pgtest to ~odssadm/dev/stoqshg/privateSettings pointing to the database on pgtest # Added stoqsadm user and created stoqs_september2012 database according to instructions in the STOQS INSTALL file # Started "~odssadm/dev/stoqshg/loaders/CANON/loadCANON_september2012.py 1 stoqs_september2012_pgtest" at 7 May 2013 1550 PDT # Monitoring vmstat 1 and iostat 1 on pgtest ## vmstat showing no swap activity. Good. ## iostat showing ~215 tps and ~3500 Blk_wrtn/s to sdc, which is /pg_xlog every second ## iostat showing 64 to 500 Blk_wrtn/s every 3 seconds or so to sdb, which is /pgdata h4. 8 May 2013 stoqs_march2013_s database copied to pgtest from odss-test by Neil using pg_dump / pg_restore to serve as a good test database that has 3,381,366 data values. Initial tests shows comparable performance between the databases accessible from the stoqs ui at http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/. Direct queries from psql on each machine are also comparable, e.g.: {noformat} On odss-test: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=629955.51..638407.16 rows=3380659 width=20) (actual time=6082.363..6971.959 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55347.59 rows=3380659 width=20) (actual time=0.020..390.344 rows=3381814 loops=1) Total runtime: 7133.318 ms (5 rows) On pgtest: stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=630174.15..638628.68 rows=3381814 width=20) (actual time=6214.177..7099.968 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external merge Disk: 112424kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..398.572 rows=3381814 loops=1) Total runtime: 7266.122 ms (5 rows) {noformat} h5. Changing shared memory from initial value of 32 MB to 2 GB (half of all physical memory) First execute script from Page 101 in Postgres 9.0 High Performance book: {noformat} <154 pgtest.shore.mbari.org /u/mccann> cat ./pg_smmem.sh #!/bin/bash # simple shmsetup script page_size=`getconf PAGE_SIZE` phys_pages=`getconf _PHYS_PAGES` shmall=`expr $phys_pages / 2` shmmax=`expr $shmall \* $page_size` echo kernel.shmmax = $shmmax echo kernel.shmall = $shmall <155 pgtest.shore.mbari.org /u/mccann> ./pg_smmem.sh kernel.shmmax = 2009198592 kernel.shmall = 490527 {noformat} Modified system and postgres .conf files: # /etc/sysctl.conf with kernel values from above # postgresql.conf with shared_buffers = 1 GB (I tried setting it to 2 GB, but got an error when restarting postgresql-9.1; our test db is only 778 MB anyway) +No joy for immediate increase in performance.+ After rebooting the "explain analyze select * from stoqs_measuredparameter order by datavalue;" query still takes about 7.5 seconds. Installed postgres contrib package in order to get tools for peeking at Postgres's memory cache: "yum install postgresql91-contrib.x86_64". Continuing to read Chapter 5... h4. 10 May 2013 Some work_mem setting tests: {noformat} stoqs_march2013_s=# show work_mem; work_mem ---------- 1MB (1 row) stoqs_march2013_s=# set work_mem='354MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=7276.353..7659.802 rows=3381814 loops=1) Sort Key: datavalue Sort Method: external sort Disk: 112400kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.048..382.085 rows=3381814 loops=1) Total runtime: 7828.067 ms (5 rows) stoqs_march2013_s=# set work_mem='355MB'; SET stoqs_march2013_s=# explain analyze select * from stoqs_measuredparameter order by datavalue; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Sort (cost=422106.15..430560.68 rows=3381814 width=20) (actual time=2399.499..2833.767 rows=3381814 loops=1) Sort Key: datavalue Sort Method: quicksort Memory: 362509kB -> Seq Scan on stoqs_measuredparameter (cost=0.00..55359.14 rows=3381814 width=20) (actual time=0.011..357.892 rows=3381814 loops=1) Total runtime: 2998.478 ms (5 rows) {noformat} +Bingo! Passing a threshold to 355MB of work_mem lets the sort happen as a quicksort in memory decreasing query time by better than half!+ Suggestions at http://stackoverflow.com/questions/8106181/work-mem-in-postgresql-on-linux and http://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ are that the application can temporarily set the work_mem to an appropriate value for the queries that need more than the default value and then reset it when the query is done. Setting it globally to a large value can cause unneeded use of work_mem leading to out of memory errors. h4. 13 May 2013 Following up on 8 May 2013 note, finishing pg_buffercache install. Note: instructions on p. 105 are not correct for pgsql-9.1. This completes the install and tests that it is installed: {noformat} postgres=# create extension pg_buffercache; CREATE EXTENSION postgres=# \dx List of installed extensions Name | Version | Schema | Description ----------------+---------+------------+--------------------------------- pg_buffercache | 1.0 | public | examine the shared buffer cache plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (2 rows) postgres=# select name,setting,unit,current_setting(name) from pg_settings where name='shared_buffers'; name | setting | unit | current_setting ----------------+---------+------+----------------- shared_buffers | 131072 | 8kB | 1GB (1 row) {noformat} h4. 14 May 2013 Following one of the suggestions from the PERFORM list, it seems that setting random_page_cost to 1 (from the default of 4) helps improve an actual query performed by the STOQS UI. With default random_page_cost = 4: {noformat} stoqs_march2013_s=# set random_page_cost=4; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------- Sort (cost=10741.41..10741.42 rows=1 width=1282) (actual time=358.452..358.452 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=3002.89..10741.40 rows=1 width=1282) (actual time=358.445..358.445 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=2983.69..10722.19 rows=3 width=954) (actual time=358.269..358.269 rows=0 loops=1) -> Nested Loop (cost=2983.69..9617.36 rows=191 width=946) (actual time=12.788..296.022 rows=20170 loops=1) -> Hash Join (cost=2983.69..8499.07 rows=193 width=842) (actual time=12.778..223.137 rows=20170 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=2982.38..8478.47 rows=4628 width=24) (actual time=12.752..188.772 rows=197746 loops=1) -> Nested Loop (cost=2982.38..4862.37 rows=512 width=4) (actual time=12.733..25.939 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.001..0.007 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=2982.38..4854.40 rows=534 width=8) (actual time=12.724..21.223 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=2982.38..2982.38 rows=534 width=0) (actual time=12.326..12.326 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..259.54 rows=12292 width=0) (actual time=5.938..5.938 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..2719.38 rows=147035 width=0) (actual time=5.847..5. 847 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..6.98 rows=7 width=20) (actual time=0.003..0. 005 rows=10 loops=20170) Index Cond: (measurement_id = mp1.measurement_id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.010..0.010 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.004..0.008 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..5.78 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=20170) Index Cond: (id = stoqs_measuredparameter.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..5.77 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone)) -> Hash (cost=18.82..18.82 rows=30 width=336) (actual time=0.165..0.165 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Hash Join (cost=1.09..18.82 rows=30 width=336) (actual time=0.073..0.162 rows=7 loops=1) Hash Cond: (stoqs_activity.platform_id = stoqs_platform.id) -> Seq Scan on stoqs_activity (cost=0.00..16.77 rows=177 width=66) (actual time=0.006..0.048 rows=177 loops=1) -> Hash (cost=1.07..1.07 rows=1 width=278) (actual time=0.018..0.018 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.012..0.015 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) Total runtime: 358.698 ms (42 rows) {noformat} With random_page_cost = 1: {noformat} stoqs_march2013_s=# set random_page_cost=1; SET stoqs_march2013_s=# explain analyze SELECT stoqs_measuredparameter.id, stoqs_parameter.name AS parameter__name, stoqs_parameter.standard_name AS parameter__standard_name, stoqs_measurement.depth AS measurement__depth, stoqs_measurement.geom AS measurement__geom, stoqs_instantpoint.timevalue AS measurement__instantpoint__timevalue, stoqs_platform.name AS measurement__instantpoint__activity__platform__name, stoqs_measuredparameter.datavalue AS datavalue, stoqs_parameter.units AS parameter__units FROM stoqs_parameter p1, stoqs_measuredparameter INNER JOIN stoqs_measurement ON (stoqs_measuredparameter.measurement_id = stoqs_measurement.id) INNER JOIN stoqs_instantpoint ON (stoqs_measurement.instantpoint_id = stoqs_instantpoint.id) INNER JOIN stoqs_parameter ON (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) INNER JOIN stoqs_activity ON (stoqs_instantpoint.activity_id = stoqs_activity.id) INNER JOIN stoqs_platform ON (stoqs_activity.platform_id = stoqs_platform.id) INNER JOIN stoqs_measuredparameter mp1 ON mp1.measurement_id = stoqs_measuredparameter.measurement_id WHERE (p1.name = 'sea_water_sigma_t') AND (mp1.datavalue > 25.19) AND (mp1.datavalue < 26.01) AND (mp1.parameter_id = p1.id) AND (stoqs_instantpoint.timevalue <= '2013-03-17 19:05:06' AND stoqs_instantpoint.timevalue >= '2013-03-17 15:35:13' AND stoqs_parameter.name IN ('fl700_uncorr') AND stoqs_measurement.depth >= -1.88 AND stoqs_platform.name IN ('dorado') AND stoqs_measurement.depth <= 83.57) ORDER BY stoqs_activity.name ASC, stoqs_instantpoint.timevalue ASC; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------------------------ Sort (cost=3930.09..3930.09 rows=1 width=1282) (actual time=185.676..185.676 rows=0 loops=1) Sort Key: stoqs_activity.name, stoqs_instantpoint.timevalue Sort Method: quicksort Memory: 25kB -> Hash Join (cost=1678.86..3930.08 rows=1 width=1282) (actual time=185.656..185.656 rows=0 loops=1) Hash Cond: (stoqs_measuredparameter.parameter_id = stoqs_parameter.id) -> Nested Loop (cost=1677.55..3928.72 rows=10 width=464) (actual time=185.626..185.626 rows=0 loops=1) -> Hash Join (cost=1677.55..3926.60 rows=1 width=456) (actual time=185.625..185.625 rows=0 loops=1) Hash Cond: (stoqs_instantpoint.activity_id = stoqs_activity.id) -> Nested Loop (cost=1667.82..3916.85 rows=7 width=128) (actual time=185.548..185.548 rows=0 loops=1) -> Nested Loop (cost=1667.82..3075.30 rows=507 width=120) (actual time=12.112..111.871 rows=20170 loops=1) -> Nested Loop (cost=1667.82..2211.81 rows=512 width=4) (actual time=12.098..26.814 rows=20170 loops=1) -> Seq Scan on stoqs_parameter p1 (cost=0.00..1.30 rows=1 width=4) (actual time=0.003..0.011 rows=1 loops=1) Filter: ((name)::text = 'sea_water_sigma_t'::text) -> Bitmap Heap Scan on stoqs_measuredparameter mp1 (cost=1667.82..2203.83 rows=534 width=8) (actual time=12.088..20.804 rows=20170 loops =1) Recheck Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision) AND (parameter_id = p1.id)) -> BitmapAnd (cost=1667.82..1667.82 rows=534 width=0) (actual time=11.707..11.707 rows=0 loops=1) -> Bitmap Index Scan on stoqs_measuredparameter_datavalue (cost=0.00..157.26 rows=12292 width=0) (actual time=5.885..5.885 r ows=23641 loops=1) Index Cond: ((datavalue > 25.19::double precision) AND (datavalue < 26.01::double precision)) -> Bitmap Index Scan on stoqs_measuredparameter_parameter_id (cost=0.00..1507.11 rows=147035 width=0) (actual time=5.265..5. 265 rows=34750 loops=1) Index Cond: (parameter_id = p1.id) -> Index Scan using stoqs_measurement_pkey on stoqs_measurement (cost=0.00..1.67 rows=1 width=116) (actual time=0.003..0.003 rows=1 loops=2017 0) Index Cond: (id = mp1.measurement_id) Filter: ((depth >= (-1.88)::double precision) AND (depth <= 83.57::double precision)) -> Index Scan using stoqs_instantpoint_pkey on stoqs_instantpoint (cost=0.00..1.65 rows=1 width=16) (actual time=0.003..0.003 rows=0 loops=20170) Index Cond: (id = stoqs_measurement.instantpoint_id) Filter: ((timevalue <= '2013-03-17 19:05:06-07'::timestamp with time zone) AND (timevalue >= '2013-03-17 15:35:13-07'::timestamp with time zone) ) -> Hash (cost=9.35..9.35 rows=30 width=336) (actual time=0.071..0.071 rows=7 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Nested Loop (cost=0.00..9.35 rows=30 width=336) (actual time=0.057..0.064 rows=7 loops=1) -> Seq Scan on stoqs_platform (cost=0.00..1.07 rows=1 width=278) (actual time=0.004..0.006 rows=1 loops=1) Filter: ((name)::text = 'dorado'::text) -> Index Scan using stoqs_activity_platform_id on stoqs_activity (cost=0.00..7.83 rows=35 width=66) (actual time=0.049..0.052 rows=7 loops=1) Index Cond: (platform_id = stoqs_platform.id) -> Index Scan using stoqs_measuredparameter_measurement_id on stoqs_measuredparameter (cost=0.00..2.02 rows=7 width=20) (never executed) Index Cond: (measurement_id = stoqs_measurement.id) -> Hash (cost=1.30..1.30 rows=1 width=826) (actual time=0.023..0.023 rows=1 loops=1) Buckets: 1024 Batches: 1 Memory Usage: 1kB -> Seq Scan on stoqs_parameter (cost=0.00..1.30 rows=1 width=826) (actual time=0.016..0.020 rows=1 loops=1) Filter: ((name)::text = 'fl700_uncorr'::text) Total runtime: 185.850 ms (40 rows) {noformat} Query went from 0.36 seconds to 0.18 seconds. Based on this the random_page_cost setting in postgresql.conf was changed from 4 to 1.01 on pgtest. Comparing similar queries between http://odss-test.shore.mbari.org/canon/stoqs_march2013_s_pgtest/query/ and http://odss-test.shore.mbari.org/canon/stoqs_march2013_s/query/ shows no appreciable improvement in performance on the pgtest database. h4. 15 May 2013 Preparing for when the STOQS application will be setting work_mem for the queries that need it I have set up logging of temp files that are created when there is not enough work_mem for sorts to be done in memory. The instructions are in the book and on this page: http://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server {quote} PostgreSQL 8.3 and later In 8.3 you can use log_temp_files to figure out if sorts are using disk instead of fitting in memory. In earlier versions, you might instead just monitor the size of them by looking at how much space is being used in the various $PGDATA/base//pgsql_tmp files. You can see sorts to disk happen in EXPLAIN ANALYZE plans as well. For example, if you see a line like "Sort Method: external merge Disk: 7526kB" in there, you'd know a work_mem of at least 8MB would really improve how fast that query executed, by sorting in RAM instead of swapping to disk. {quote} The change to the postgresql.conf file was made today with the previous version saved to postgresql.conf.130515 ]]> 21364738 21398451 kraken_sar_1aug2013.txt {noformat} [^kraken_sar_1aug2013.txt] The highest %util was 12.59% at 2:20 PM. We're barely scratching the surface with the capabilities of this machine. ]]> 21365738 9404710 9371961 10387866 10355129 9404766 9372018 9404770 9372024 9404769 9372023 9404767 9372021 9404762 9372014 21434014 21365564 2013-07-23 11:29:59.887 2013-07-23 11:29:59.887 9404750 MBnav created and first navigation files edited:Navigation editing initially done by IAG: John Graybeal, Andrew Chase\\ Navigation editing by ITD:  March(?) 2006,    Jane Curtner and Connor                                          September 2007, Linda Kuhnz, Lori Chaney, Dana Lacono ]]> 9372001 9404753 9372005 9404751 9372003 9404746 9371997 9404795 9372049 9404797 9372051 9404800 9372054 9404799 9372053 21433991 21365488 2013-07-17 16:21:56.093 2013-07-17 16:21:56.093 21433992 21365523 2013-07-17 16:28:55.120 2013-07-17 16:28:55.120 21433993 21365523 2013-07-17 16:28:55.120 2013-07-17 16:28:55.120 9404791 9372045 9404793 *Current Issues:*{color:red} 02-March-2009\- L. Kuhnz.  Doc Ricketts files cannot  be edited.{color}To edit Doc Rickets dives, we must enter "165" as a format ID (in the file, open dialog box).  This will be corrected in the next MBnav release per Dave Caress. The output file is not created correctly; waiting for instructions from Dave Caress. \\ *Historic issues:*1) Between {color:red}26 June 2008 and 03 February 2009{color}, there is a roundoff error logged in all raw navigation data files essentially rendering the last 2 digits of latitude and longitude meaningless.The truncation of precision was because of error in the linux port of the datamanager code. The error was identified and fixed on 3 Feb 2009. An analysis of Steve Rock's Ventana dive V3241 (a video mosaic survey) by using data from the winfroglogr files (rov nav input to datamanager) and properly edited data (from the low precision archived nav) indicates that very little improvement in the final edited product can be had by replacing the low precsion nav with nav reprocessed from the winfroglor data.2) In 2001, there was a navigation "freezing" problem.  This problem occurred from {color:red}March 25th, 2001 11:59:58 PM through{color} {color:red}December 14th, 2001 05:10:23 AM. {color}{color:black}See John Graybeal's .ppt presentation about this issue (Sept 2001){color} {color:red}[^NavProblemsReview_Graybeal.ppt]{color}3) There was an issue with dives {color:red}T119 to T244{color}whereby ships navigation was written to the ROV raw navigation files;  ROV navigation for these dives is therefore erroneous. See attached excel sheet [^CompareWflyTibr-Graybeal.xls]{color:#000000}and the attached{color} {color:#000000}[{color:#000000}map{color}|NavComparison_2000_Davidson-newEdNav.jpg]{color} {color:#000000}(provided by Jenny Paduan) for a visual example of the problem.{color}4) Related to #3: As of 12-February 2009, Dave Clague and Jenny Paduan are working with Mike McCann to use the Markpoint files created during these dives:  {color:red}T119-124, T139-147, T185-201, and T207-208{color} to produce smoothed navigation files to merge with the video annotations. This series of dives did not have an ROV nav file recorded since ship nav was incorrectly being placed in the ROV nav file. They are going to use Markpoint files to reconstruct the ROV path. Note that the following files are now pointing to markpoint data: nav2000081tibredited.txt - nav2000086tibredited.txt nav2000127tibredited.txt - nav2000130tibredited.txt nav2000133tibredited.txt nav2000135tibredited.txt - nav2000136tibredited.txt nav2000219tibredited.txt - nav2000233tibredited.txt nav2000243tibredited.txt - nav2000244tibredited.txt They are also looking at the navigation of dive {color:red}T425{color} with Mike and trying to adjust the track so that it better fits the bathymetry. At present, although the dive plots as starting and ending at the fish trap (located during navigated dive {color:red}T427{color}), it is not plotting so that depths match with the underlying bathymetry. This is the dive that did not have any USBL navigation because the transponder on the vehicle was not operational.5) Prior to September 2005, VARS (and earlier VIMS) database queries returned navigation data from the unedited navigation files.  ]]> 9372047 9404779 9372033 33161220 cd /mbari/Tempbox/mccann <103 elvis.shore.mbari.org /Tempbox/mccann> tar cvf 2012.268.07.tar 2012.268.07 2012.268.07/ 2012.268.07/SmartSamplerDataLog.log 2012.268.07/Gulper.log ... 2012.268.07/ctdDriver2.cfg 2012.268.07/gps.log 2012.268.07/m3dmgx1.log <105 elvis.shore.mbari.org /Tempbox/mccann> gzip 2012.268.07.tar {noformat} # Copy tarball to /usr/local/auv_ctd and run the auvportal which submits the data to SSDS: {noformat} <255 elvis.shore.mbari.org /Tempbox/mccann> cp 2012.268.07.tar.gz /usr/local/auv_ctd <256 elvis.shore.mbari.org /Tempbox/mccann> cd /usr/local/auv_ctd <257 elvis.shore.mbari.org /local/auv_ctd> ./portal AUVCTD2012268 2012.268.07.tar.gz Tue, 25 Sep 2012 09:27:43 PDT: processConnection() got message: AUVCTD2012268;2012.268.07.tar.gz dstDir.getName() = 2012268 Unpacking /usr/local/auv_ctd/2012.268.07.tar.gz Copying /usr/local/auv_ctd/2012268/2012.268.07/ctdDriver2.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/ctdDriver2.log Copying /usr/local/auv_ctd/2012268/2012.268.07/gps.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/gps.log Copying /usr/local/auv_ctd/2012268/2012.268.07/isuslog.log to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/isuslog.log Copying /usr/local/auv_ctd/2012268/2012.268.07/devices.cfg to /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07/devices.cfg ... DEBUG [Thread-0] (XmlBuilder.java:538) - appendPrimitiveType called with method name isValidRole on object of class moos.ssds.metadata.DataProducer DEBUG [Thread-0] (XmlBuilder.java:482) - appendElement called on class moos.ssds.metadata.DataProducer using method getClass DEBUG [Thread-0] (XmlBuilder.java:278) - Converting java.lang.Class(class moos.ssds.metadata.DataProducer) to XML Marshalled the AUV mission in /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 into XML Attempting to publish AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ published XML describing mission AUVCTD /mbari/AUVCTD/missionlogs/2012/2012268/2012.268.07 to SSDS server jnp://bob.shore.mbari.org:1099/ If everything goes well in SSDS:Ingest/Ruminate an email message to auvctd@listserver.mbari.org will arrive shortly. Tue, 25 Sep 2012 09:34:29 PDT: Done receiving AUV mission data. {noformat} Note: With the move of the portal to from jeckel to elvis the mailing to auvctd@listserver.mbari.org has stopped working. This needs to be debugged and fixed. # Process the LOPC data, e.g.: {noformat} <109 elvis.shore.mbari.org /u/ssdsadmin> cd ~/dev/auv_ctd/src/py/auv/ <109 elvis.shore.mbari.org /u/ssdsadmin> /bin/bash <109 elvis.shore.mbari.org /u/ssdsadmin> export PATH="/usr/local/miniconda2/bin:$PATH" <109 elvis.shore.mbari.org /u/ssdsadmin> source activate auv_ctd (auv_ctd) bash-4.2$ processLOPC.pl 2013.224.02 (auv_ctd) bash-4.2$ exit {noformat} # Once the mission's instrument netcdf files have been created (check pismo:/opt/ssds/logs/ssds-updateBot.log for successful processing) the overall Dorado science data processing script can be edited with the new survey to be processed: {noformat} <102 elvis.shore.mbari.org /u/ssdsadmin> cd ~/dev/auv_ctd/src/matlab/doradosdp/process/ <103 elvis.shore.mbari.org /doradosdp/process> vi reprocess_surveys.m {noformat} # Add a line for the new survey to the end of the reprocess_surveys.m file, commenting out the previous survey that was processed: {noformat} % CANON September 2012 - laps around ESP drifter - last one for this campaign %%AUVprocess_main('AUVCTD', 2012, ['258.00'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') % CANON September 2012 - laps around ESP drifter - last one for this campaign AUVprocess_main('AUVCTD', 2012, ['268.07'], server, ssdsLoadFlag, ssdsLoadsJar, localFlag, localDataDir, localCalDir, notifyEmail, skipMissionProc, [], 'ctd1') {noformat} # Execute the Matlab job with a shell script that properly sets the DISPLAY environment and executes reprocess_surveys.m with matlab: {noformat} <104 elvis.shore.mbari.org /doradosdp/process> RunReprocessJob {noformat} # Check for error messages. If everything works an email is sent to the notifyEmail setting in reprocess_surveys.m, usually auvctd@listserver.mbari.org. h2. Software checkout and repository code management The Software for both the AUV Portal and the Science data processing are located in the _auv_ctd_ module on the CVS server moonjelly.shore.mbari.org. To check the software to a system use your favorite CVS client, or on Unix use these commands: {noformat} cd cvs -d :pserver:@moonjelly:/home/cvs login (enter cvs account password) cvs -d :pserver:@moonjelly:/home/cvs co -d auv_ctd -P auv_ctd (about a minute wait for the files to be put in your directory) {noformat} (Replace with a subdirectory name of your choice and with your moonjelly login name.) If you have already done this software checkout step and want to update your copy of the software from the repository (which you would want to do if someone else on the team has commited some changes to the repository) then you can get these changes with the cvs update command, .e.g.:cd /auv_ctd cvs update(Note that if new directories have been added to the repository you will need to repeat the above checkout command.) If you have made changes in the software and wish to submit them back to the repository for others to use then use the cvs commit command, e.g.:cvs commit (your default editor will open the files that have been changed for you to enter text describnig the changes you made)   h2. AUV Portal h3. Build and start-up The AUV Portal is the software that accepts data collected by the AUV. It constructs SSDS-compliant metadata, submits this mission metadata to SSDS and copies the data (.log and .cfg files) to the shore-side network storage location. After checking out the software from CVS, build it with Ant and run it (Note: you will need both Apache Ant and Java JDK 1.4 or higher on your system with the ANT_HOME and JAVA_HOME environment variables set). {noformat} Build it (ie. Type 'ant') cd to the build/dist directory type 'portal' (windows or UNIX): {noformat} {noformat} > cd auv_ctd > ant (wait for a BUILD SUCCESSFUL message) > cd build\dist > portal > java \-cp .\activation.jar;.\auvctd.jar;.\ftp.jar;.\jbossall-client.jar;.\junit.jar;.\log4j.jar;.\siam.jar; .\ssds.jar;.\tar.jar;.\xercesImpl.jar;.\xmlParserAPIs.jar;.\xom.jar;. moos.operations.portal.AUVPortalService Wait for client... {noformat} h3. AUV data submission to SSDS To submit AUV mission data to SSDS, log into the vehicle computer and use the adx command to transfer the data to the AUV Portal that you have started with the above directions. The adx syntax is as follows: {noformat} usage: adx PortalName DataName login pass survey dir \[dir2 ...\] PortalName is the hostname of the portal DataName is the hostname where the auv data resides login is the login account on DataName that the portal will use pass is the password to the login on DataName SSDS survey name (e.g., AUVCTD_2003_191) dir is the FULL path name of the data directory to copy {noformat} For example, if auv-2-control is the name of the computer running the Portal, mvc-dmo1 is the name of the AUV's computer, login1 and pass1 are the login name and password for the AUV computer, AUVCTD2004028 is the survey name, and /home/dorado1/auv/altex/onboard/logs/2004.028.05 is the location of the data files on the AUV then the command would be: {noformat} adx auv-2-control mvc-dmo1 login1 pass1 AUVCTD2004028 /home/dorado1/auv/altex/onboard/logs/2004.028.05 {noformat} Here is a diagram representing the data flow from AUV through the Portal to SSDS. (Option 2 is what is currently used by the auv_ctd Portal.) | | | *Figure 1.* Diagram of data flow between AUV, AUV Portal, and Shore Side Data System. | On the AUV computer (where you ran the adx command the screen will look something like this: {noformat} bash-2.00$ adx 134.89.13.105 mvc-dmo1 dorado1 dorado1 OCCO2003237 /home/dorado1/auv/altex/onboard/logs/2003.237.04 /home/dorado1/auv/altex/onboard/logs/2003.237.04 ; cd /home/dorado1/auv/altex/onboard/logs ... ; tar cvf 2003.237.04_data.tar 2003.237.04 ... a 2003.237.04/abortMplan.cfg 1 Blocks a 2003.237.04/bluefinBattlog.log 524 Blocks a 2003.237.04/crossbow.log 35615 Blocks a 2003.237.04/ctdDriver.cfg 2 Blocks a 2003.237.04/ctdDriver.log 11179 Blocks a 2003.237.04/ctdDriver2.cfg 2 Blocks a 2003.237.04/ctdDriver2.log 11179 Blocks a 2003.237.04/devices.cfg 4 Blocks a 2003.237.04/dvl.cfg 7 Blocks a 2003.237.04/dvl.log 5323 Blocks a 2003.237.04/DvlData.bin 33617 Blocks a 2003.237.04/dynamicControl.log 25404 Blocks a 2003.237.04/events.log 0 Blocks a 2003.237.04/gps.log 69 Blocks a 2003.237.04/hydroscatlog.log 7627 Blocks a 2003.237.04/isuslog.log 610 Blocks a 2003.237.04/lisstlog.log 8626 Blocks a 2003.237.04/missionfile.cfg.pp 11 Blocks a 2003.237.04/navigation.log 31120 Blocks a 2003.237.04/parosci.log 7632 Blocks a 2003.237.04/syslog 11231 Blocks a 2003.237.04/tailCone.log 7630 Blocks a 2003.237.04/vehicle.cfg 7 Blocks a 2003.237.04/workSite.cfg 4 Blocks a 2003.237.04/2003.237.04_data.tar.gz 71527 Blocks ; gzip \-f 2003.237.04_data.tar ... ; mv 2003.237.04_data.tar.gz 2003.237.04/. ... ; Portal message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data .tar.gz bash-2.00$ {noformat} \\ On the portal computer the screen will display text something like this: {noformat} Wait for client... accepted connection processConnection() - end-of-stream Wait for client... processConnection() got message: 134.89.32.26;dorado1;dorado1;OCCO2003237;/home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieve mission files from AUV host: 134.89.32.26: /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz Retrieving /home/dorado1/auv/altex/onboard/logs/2003.237.04/2003.237.04_data.tar.gz All done dstDir.getName() = OCCO2003237 Unpacking C:\dev\auv_ctd\build\dist\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\2003.237.04_data.tar.gz Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\abortMplan.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\abortMplan.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\bluefinBattlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\bluefinBattlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\crossbow.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\crossbow.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.cfg\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\ctdDriver2.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\ctdDriver2.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\devices.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\devices.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dvl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dvl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\DvlData.bin to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\DvlData.bin \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\dynamicControl.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\events.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\gps.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\gps.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\hydroscatlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\hydroscatlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\isuslog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\isuslog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\lisstlog.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\lisstlog.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\missionfile.cfg.pp to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\missionfile.cfg.pp \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\navigation.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\navigation.log\[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\parosci.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\parosci.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\syslog to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\syslog \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\tailCone.log to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\tailCone.log \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\vehicle.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\vehicle.cfg \[WARNING\!\! Overwriting existing file\] Copying C:\dev\auv_ctd\build\dist\OCCO2003237\2003.237.04\workSite.cfg to \\Tornado\AUVCTD\OCCO2003237\2003.237.04\workSite.cfg \[WARNING\!\! Overwriting existing file\] [http://ssds.shore.mbari.org/auvctd/OCCO2003237/2003.237.04] WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\dynamicControl.log WARNING\!\! Device information is missing from \\Tornado\AUVCTD\OCCO2003237\2003.237.04\events.log moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING moos.ssds.model.XmlBuilder: Attempted to call "public java.util.Collection moos.ssds.model.Deployment.listChildDeployments(java.lang.String)", an access method that requires an argument, SKIPPING Marshalled the AUV mission in \\Tornado\AUVCTD\OCCO2003237\2003.237.04 into XML Attempting to publish \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS. SUCCESS\!\! published \\Tornado\AUVCTD\OCCO2003237\2003.237.04 to SSDS {noformat} Note: Detailed command-line instructions for "portaling" AUVCTD mission data into SSDS are available in the README file in the project or in the operational location on jeckel.shore.mbari.org in /usr/local/auv_ctd/ directory. There is also the portal.log file there which logs the output of the deamon process running on jeckel that receives and processes the data from AUV uploads from the vehicle. The portal submission process creates self-described netCDF files of all the logged data from each instrument. To produce more useful data products the science data processing must be done; this is described in the next section. h1. Dorado Science Data Processing Within the auv_ctd software module is the auv_ctd/src/matlab/process directory. This directory contains the m-files for processing the AUV instrument data such that they may be easily used for scientific investigation. The products include: * "level-1" data = calibrated & aligned, in native (full) resolution. This is the basis for the quick-look section plots. * "level-2" data = level-1 decimated to limiting resolution (ISUS) so that we can serve data as \[time, lat, lon, depth, variable 1:N\] which is commonly needed. A "byproduct" of the processing is the generation of processing metadata which is loaded into SSDS so that we can track data products and the software that produce them. Here is a graph snippet showing the level-1 and level-2 data products for an AUV survey To produce these data products check out the auv_ctd software as described above to a system that has Matlab version 6 or higher. From the auv_ctd/src/matlab/doradosdp/process directory start Matlab and execute the AUVprocess_main script. The documentation for this script is: {noformat} > matlab \-nodesktop < M A T L A B > Copyright 1984-2002 The MathWorks, Inc. Version 6.5.0.180913a Release 13 Jun 18 2002 To get started, type one of these: helpwin, helpdesk, or demo. For product information, visit www.mathworks.com. >> help AUVprocess_main %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % AUVprocess_main.m: front end script to be modified by the AUV % science data processing person or in an automated way by SSDS. % % $Id: ScienceDataProcessing.html,v 1.3 2004/05/19 23:17:52 mccann Exp $ % % Usage: % AUVprocess_main \[ , , , , .. % , , , \] % % campaign: Name of particular type of missions (e.g. 'AUVCTD', 'BIOLUM', 'OCCO') % year: Year of the missions % start-year-day.mission: First mission to use for the section (e.g. '338.15') % end-year-day.mission: Last mission to use for the section (e.g. '338.18') % ssds_load_flag: If 1 then load processing metadata into SSDS % ssdsLoads_jar: Name of SSDS loads jar file, (e.g. ssdsLoads-prey.jar) % local_flag: Set to 0 to use SSDS DODS urls for input data, 1 for local files % local_dir: If local_flag is 1 then use to read the input data % % % % Examples: % To process data from SSDS DODS-accessible datasets, loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 1, 'ssdsLoads-prey.jar', 0, '') % % To process data from local files, not loading SSDS metadata: % >> AUVprocess_main('BIOLUME', 2003, '338.15', '338.18', 0, '', 1, '/hosts/tornado/auvctd/') % % Year-days must have leading zeros, i.e. they must be 3 digits. % % % Files copied to web accessible locations by this function include: % - Matlab data file (.mat) of the constructed survey % - A png image file that is a summary of the data in the .mat file % - An ODV compliant spreadsheet file (.txt) % - A COARDS-compliant DODS accessible netcdf file (.nc) that contains the same data of the .mat file % - A log file (diary of this script) % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% >> {noformat} So to process a set of missions into a survey execute with the start and end mission and the proper parameters for accessing the data and loading the metadata into SSDS. For example this command creates a survey from 3 missions using the DODS accessible netCDF files and loads the data into the SSDS server prey: >> AUVprocess_main('BIOLUME', 2003, '245.02', '245.04', 1, 'ssdsLoads-prey.jar', 0, '') Will produce a bunch of output to the screen, which is captured as a diary log file and saved in SSDS. When it is finished you should see the last few lines like: {noformat} Submitting metadata to SSDS... done. ProcessRun id = 1802 Processing time: 289 seconds {noformat} +Important note+: If changes are made to any of the matlab m-files they should be checked back into the CVS repository before the AUVprocess_main script is run. This ensures that the ProcessRun Software and Resources will have the correct version number references in SSDS. It's easy to commit the changes. Just run 'cvs commit' from the Unix command line before running AUVprocess_main. Two level 2 products produced by this processing step are a netCDF file of the aligned and ISUS time step sub-sampled variables and an Ocean Data View spreadsheet file. The ODV Spreadsheet file is not as efficient or as capable as a native ODV Collection (a .var file). If one wishes to work with AUV survey data in ODV then all the files for a survey in \\tornado\AUVCTD\surveys\\odv should be copied to a local directory so that one has write permission to the files.]]> 33095684 9404786 9372040 9830430 9797662 9404774 9372028 11272716 11239968 8946028 8913275 21398133 21365410 21398125 21365401 2 2006-10-17 21:48:40.373 2026-02-03 16:16:06.820 21398209 diff sysctl.conf.130712 sysctl.conf 40a41,53 > > # Swappiness set to 0 per p.81 in PostgreSQL 9.0 High Performance book. mpm - 12 July 2013 > vm.swappiness = 0 > # Reduce tendency to allocate more ram than the system has. mpm - 12 July 2013 > vm.overcommit_memory = 2 > > # Setting for shared memory = 1/2 physical RAM = 64 GB. mpm 12 July 2013 > ##kernel.shmmax = 67649847296 > ##kernel.shmall = 16516076 > > # Set shared memory to 4 GB - a better value for PostgreSQL on large memory systems. mpm 12 July 2013 > kernel.shmmax = 4228112384 > kernel.shmall = 1032254 <198 kraken.shore.mbari.org //etc> diff fstab.130712 fstab 11,13c11,13 < /dev/mapper/vg_kraken-LogVol02 /pg_xlog_spinning ext4 defaults 1 2 < /dev/mapper/vg_kraken-LogVol04 /pgdata_spinning ext4 defaults 1 2 < /dev/mapper/vg_kraken-LogVol03 /tablespace_spinning ext4 defaults 1 2 --- > /dev/mapper/vg_kraken-LogVol02 /pg_xlog_spinning ext4 defaults,noatime 1 2 > /dev/mapper/vg_kraken-LogVol04 /pgdata_spinning ext4 defaults,noatime 1 2 > /dev/mapper/vg_kraken-LogVol03 /tablespace_spinning ext4 defaults,noatime 1 2 22,24c22,24 < /dev/mapper/iomemory_vg-iomemVol1 /pg_xlog ext4 defaults,noauto,discard 0 0 < /dev/mapper/iomemory_vg-iomemVol2 /tablespace ext4 defaults,noauto,discard 0 0 < /dev/mapper/iomemory_vg-iomemVol3 /pgdata ext4 defaults,noauto,discard 0 0 --- > /dev/mapper/iomemory_vg-iomemVol1 /pg_xlog ext4 defaults,noatime,noauto,discard 0 0 > /dev/mapper/iomemory_vg-iomemVol2 /tablespace ext4 defaults,noatime,noauto,discard 0 0 > /dev/mapper/iomemory_vg-iomemVol3 /pgdata ext4 defaults,noatime,noauto,discard 0 0 <199 kraken.shore.mbari.org //etc> diff rc.local.130712 rc.local 7a8,9 > blockdev --setra 8192 /dev/mapper/vg_kraken-LogVol04 > blockdev --setra 8192 /dev/mapper/iomemory_vg-iomemVol3 {noformat} ]]> 21365487 21398211 # Allow connections from mbari hosts inside our firewall - mpm 7 May 2013 > listen_addresses = '*' # what IP address(es) to listen on; 64c65,66 < max_connections = 100 # (change requires restart) --- > # Changed max_connections from 100 to 10 corresponding to increase in work_mem - mpm 22 May 2013 > max_connections = 10 # (change requires restart) 109c111 < shared_buffers = 32MB # min 128kB --- > shared_buffers = 4GB # min 128kB 118c120,122 < #work_mem = 1MB # min 64kB --- > > # Changed from 1 MB to 400MB to support large STOQS sorts - mpm 22 May 2013 > work_mem = 400MB # min 64kB 174c178,179 < #checkpoint_segments = 3 # in logfile segments, min 1, 16MB each --- > # pgbench-tools indicated high checkpoint frequency, adjust with guidance from p.137. dr 3 July 2013 > checkpoint_segments = 10 # in logfile segments, min 1, 16MB each 176c181 < #checkpoint_completion_target = 0.5 # checkpoint target duration, 0.0 - 1.0 --- > checkpoint_completion_target = 0.9 # checkpoint target duration, 0.0 - 1.0 244c249,251 < #random_page_cost = 4.0 # same scale as above --- > > # Changed to 1.01 per Greg Smith's suggestion on postgresql-perform list - seems to improve STOQS mp query performance - mpm 14 May 2013 > random_page_cost = 1.01 # same scale as above 359c366,367 < #log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements --- > # Log all queries taking longer than 1 second. mpm 15 July 2013 > log_min_duration_statement = 1000 # -1 is disabled, 0 logs all statements 377c385,387 < #log_line_prefix = '' # special values: --- > # Set timestamp for log files to be compatible with pgFouine. mpm 15 July 2013 > log_line_prefix = '%t [%p]: [%l-1] user=%u, db=%d' > # special values: 399c409,410 < #log_temp_files = -1 # log temporary files equal or larger --- > # Turned on logging of temp files. See $PGDATA/base//pgsql_tmp for what a DB is doing - mpm 15 May 2013 > log_temp_files = 0 # log temporary files equal or larger 482c493,494 < #timezone = '(defaults to server environment setting)' --- > # Severs running STOQS databases must have timezone set to 'GMT' > timezone = 'GMT' {noformat} ]]> 21365489 21398268 21365548 12058625 2010-04-28 15:37:00.777 2015-07-30 11:42:03.520 21398244 # Allow connections from mbari hosts inside our firewall - mpm 7 May 2013 > listen_addresses = '*' # what IP address(es) to listen on; 64c65,66 < max_connections = 100 # (change requires restart) --- > # Changed max_connections from 100 to 10 corresponding to increase in work_mem - mpm 22 May 2013 > max_connections = 10 # (change requires restart) 109c111 < shared_buffers = 32MB # min 128kB --- > shared_buffers = 3GB # min 128kB 118c120,122 < #work_mem = 1MB # min 64kB --- > > # Changed from 1 MB to 400MB to support large STOQS sorts - mpm 22 May 2013 > work_mem = 400MB # min 64kB 174c178,179 < #checkpoint_segments = 3 # in logfile segments, min 1, 16MB each --- > # pgbench-tools indicated high checkpoint frequency, adjust with guidance from p.137. dr 3 July 2013 > checkpoint_segments = 10 # in logfile segments, min 1, 16MB each 176c181 < #checkpoint_completion_target = 0.5 # checkpoint target duration, 0.0 - 1.0 --- > checkpoint_completion_target = 0.9 # checkpoint target duration, 0.0 - 1.0 244c249,251 < #random_page_cost = 4.0 # same scale as above --- > > # Changed to 1.01 per Greg Smith's suggestion on postgresql-perform list - seems to improve STOQS mp query performance - mpm 14 May 2013 > random_page_cost = 1.01 # same scale as above 359c366,367 < #log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements --- > # Log all queries taking longer than 1 second. mpm 15 July 2013 > log_min_duration_statement = 1000 # -1 is disabled, 0 logs all statements 377c385,387 < #log_line_prefix = '' # special values: --- > # Set timestamp for log files to be compatible with pgFouine. mpm 15 July 2013 > log_line_prefix = '%t [%p]: [%l-1] user=%u, db=%d' > # special values: 399c409,410 < #log_temp_files = -1 # log temporary files equal or larger --- > # Turned on logging of temp files. See $PGDATA/base//pgsql_tmp for what a DB is doing - mpm 15 May 2013 > log_temp_files = 0 # log temporary files equal or larger 482c493,494 < #timezone = '(defaults to server environment setting)' --- > # Severs running STOQS databases must have timezone set to 'GMT' > timezone = 'GMT' {noformat} ]]> 21365522 21398290 21365570 21398285 ;date.timezone = {noformat} Run a pgbench test to generate some query traffic. {noformat} -bash-4.1$ ./pgbench -T 60 -c 4 pgbench starting vacuum...end. transaction type: TPC-B (sort of) scaling factor: 160 query mode: simple number of clients: 4 number of threads: 1 duration: 60 s number of transactions actually processed: 34324 tps = 571.932873 (including connections establishing) tps = 572.082084 (excluding connections establishing) {noformat} Run pgfouine on todays log file to make sure it works. {noformat} ##### Overall statistics ##### Number of unique normalized queries: 1 Number of queries: 1 Total query duration: 0.0s ##### Queries by type ##### SELECT: 1 100.0% ##### Queries that took up the most time (N) ##### 1) 0.0s - 1 - select count(*) from pgbench_branches -- ##### Slowest queries ##### 1) 0.00 s - select count(*) from pgbench_branches -- ##### Most frequent queries (N) ##### 1) 1 - 0.0s - select count(*) from pgbench_branches -- ##### Slowest queries (N) ##### 1) 0.00 - 1 - select count(*) from pgbench_branches -- {noformat} ]]> 21365565 21398286 ;date.timezone = {noformat} Run a pgbench test to generate some query traffic. {noformat} -bash-4.1$ ./pgbench -T 60 -c 4 pgbench starting vacuum...end. transaction type: TPC-B (sort of) scaling factor: 160 query mode: simple number of clients: 4 number of threads: 1 duration: 60 s number of transactions actually processed: 34324 tps = 571.932873 (including connections establishing) tps = 572.082084 (excluding connections establishing) {noformat} Run pgfouine on todays log file to make sure it works. *note: The command starts with pgfouine.php not just pgfouine. {noformat} -bash-4.1$ pgfouine.php -logtype stderr -file /var/lib/pgsql/9.1/data/pg_log/postgres-Wed.log ##### Overall statistics ##### Number of unique normalized queries: 1 Number of queries: 1 Total query duration: 0.0s ##### Queries by type ##### SELECT: 1 100.0% ##### Queries that took up the most time (N) ##### 1) 0.0s - 1 - select count(*) from pgbench_branches -- ##### Slowest queries ##### 1) 0.00 s - select count(*) from pgbench_branches -- ##### Most frequent queries (N) ##### 1) 1 - 0.0s - select count(*) from pgbench_branches -- ##### Slowest queries (N) ##### 1) 0.00 - 1 - select count(*) from pgbench_branches -- {noformat} ]]> 21365566 21398292 diff /etc/php.ini.130723 /etc/php.ini 946c946 < ;date.timezone = --- > date.timezone = America/Los_Angeles {noformat} Create a web directory for pgFouine reports and test running it: {noformat} sudo mkdir /var/www/html/pgfouine sudo chown postgres /var/www/html/pgfouine pgfouine.php -logtype stderr -file /var/lib/pgsql/9.1/data/pg_log/postgresql-Mon.log -format html > /var/www/html/pgfouine/InitialTest.html {noformat} Results viewable at http://kraken.shore.mbari.org/pgfouine/InitialTest.html ]]> 21365572 21398291 diff /etc/php.ini.130723 /etc/php.ini 946c946 < ;date.timezone = --- > date.timezone = America/Los_Angeles {noformat} Create a web directory for pgFouine reports and test running it: {noformat} sudo mkdir /var/www/html/pgfouine sudo chown postgres /var/www/html/pgfouine pgfouine.php -logtype stderr -file /var/lib/pgsql/9.1/data/pg_log/postgresql-Mon.log -format html > /var/www/html/pgfouine/InitialTest.html {noformat} ]]> 21365571 21398293 diff /etc/php.ini.130723 /etc/php.ini 946c946 < ;date.timezone = --- > date.timezone = America/Los_Angeles {noformat} Create a web directory for pgFouine reports and test running it: {noformat} sudo mkdir /var/www/html/pgfouine sudo chown postgres /var/www/html/pgfouine pgfouine.php -logtype stderr -file /var/lib/pgsql/9.1/data/pg_log/postgresql-Mon.log -format html > /var/www/html/pgfouine/InitialTest.html {noformat} Results viewable at http://kraken.shore.mbari.org/pgfouine/InitialTest.html ]]> 21365573 21398304 log_filename = 'postgresql-%Y%m%d.log' # log file name pattern, 367c367 < log_min_duration_statement = 100 # -1 is disabled, 0 logs all statements --- > log_min_duration_statement = 10 # -1 is disabled, 0 logs all statements {noformat} pgFouine now: {noformat} pgfouine.php -logtype stderr -file /var/lib/pgsql/9.1/data/pg_log/postgresql-20130723.log -format html > /var/www/html/pgfouine/postgresql-20120723.html {noformat} gives: http://kraken.shore.mbari.org/pgfouine/postgres-20120723.html The pgfouine.php command is something we can put into cron, or just run it when we need to. ]]> 21365584 21398305 log_filename = 'postgresql-%Y%m%d.log' # log file name pattern, {noformat}]]> 21365585 21398437 log_filename = 'postgresql-%Y%m%d.log' # log file name pattern, 367c367 < log_min_duration_statement = 100 # -1 is disabled, 0 logs all statements --- > log_min_duration_statement = 10 # -1 is disabled, 0 logs all statements {noformat} pgFouine now: {noformat} pgfouine.php -logtype stderr -file /var/lib/pgsql/9.1/data/pg_log/postgresql-20130723.log -format html > /var/www/html/pgfouine/postgresql-20130723.html {noformat} gives: http://kraken.shore.mbari.org/pgfouine/postgresql-20120723.html The pgfouine.php command is something we can put into cron, or just run it when we need to. ]]> 21365722 21398436 log_filename = 'postgresql-%Y%m%d.log' # log file name pattern, 367c367 < log_min_duration_statement = 100 # -1 is disabled, 0 logs all statements --- > log_min_duration_statement = 10 # -1 is disabled, 0 logs all statements {noformat} pgFouine now: {noformat} pgfouine.php -logtype stderr -file /var/lib/pgsql/9.1/data/pg_log/postgresql-20130723.log -format html > /var/www/html/pgfouine/postgresql-20120723.html {noformat} gives: http://kraken.shore.mbari.org/pgfouine/postgresql-20120723.html The pgfouine.php command is something we can put into cron, or just run it when we need to. ]]> 21365721 21398452 21365740 21398481 log_filename = 'postgresql-%Y%m%d.log' # log file name pattern, 367c367 < log_min_duration_statement = 100 # -1 is disabled, 0 logs all statements --- > log_min_duration_statement = 10 # -1 is disabled, 0 logs all statements {noformat} pgFouine now: {noformat} pgfouine.php -logtype stderr -file /var/lib/pgsql/9.1/data/pg_log/postgresql-20130723.log -format html > /var/www/html/pgfouine/postgresql-20130723.html {noformat} gives: http://kraken.shore.mbari.org/pgfouine/postgresql-20130723.html The pgfouine.php command is something we can put into cron, or just run it when we need to. ]]> 21365769 8913211 5 false 0.0 0 0 9371817 5 false 0.0 0 0 8913216 5 false 0.0 0 0 9371824 5 false 0.0 0 0 21365241 5 false 0.0 0 0 16777434 6 false 0.0 0 0 21365241 6 false 0.0 0 0 9372027 6 false 0.0 0 0 8913230 5 false 0.0 0 0 9372235 6 false 0.0 0 0 9372235 5 false 0.0 0 0 8913263 6 false 0.0 0 0 9372068 5 false 0.0 0 0 9371973 5 false 0.0 0 0 9371999 6 false 0.0 0 0 9372076 5 false 0.0 0 0 8913230 6 false 0.0 0 0 9372027 5 false 0.0 0 0 20971983 6 false 0.0 0 0 8913216 6 false 0.0 0 0 8913263 5 false 0.0 0 0 8913211 6 false 0.0 0 0 1179693 5 false 0.0 0 0 9372076 6 false 0.0 0 0 8913167 5 false 0.0 0 0 8913162 5 false 0.0 0 0 9371973 6 false 0.0 0 0 9371824 6 false 0.0 0 0 9371817 6 false 0.0 0 0 20971983 5 false 0.0 0 0 5374441 5 false 0.0 0 0 16777434 5 false 0.0 0 0 9371999 5 false 0.0 0 0 9372068 6 false 0.0 0 0 5374441 6 false 0.0 0 0 1179693 6 false 0.0 0 0 8913167 6 false 0.0 0 0 8913162 6 false 0.0 0 0