% update_GliderVIZ_sat.m % % Step 1: download .sat file from sftp. Compare it to existing file. If the % file is bigger, process. if same, do nothing. If smaller, means it caught % the file while server was updating. Just process it the next time around. % % update notes % % 10/11/2020 % - added copy files to atlas (raw sat files, processed files, kml files) % % 12/4/2020 % - changed approach for downloading data from server. switched from sftp_simple_get (downloaded % from fileshare), which used a Ganymded SSH-2 toolbox, which was not installed on the recent Ubuntu % update for the bog-fire server (summer 2020). So now use a command line approach with putty to % send the password through. clear close all % ********** USER INPUT ************************ % activemissions = {'20A02901' % '20A03401'}; force_process = false; % debugging boolean. If true, it will process sat file % activemissions = {}; % If no mission, keep it empty activemissions = {'22303401'}; % activemissions = {}; % activemissions = {'21202901', '21503401'};%, '21303401'}; % *********************************************** % ======================== MAIN ROUTINE START ============================= % directory paths to archive data % !! IF HAVING ISSUES DUING NEXT DEPLOYMENT, CHANGE THIS BACK TO DOPBOX % depdirectory = 'C:\Users\yui\Dropbox\Spray_pH\Deployments\'; depdirectory = '\\atlas\projectlibrary\901805_Coastal_Biogeochemical_Sensing\Spray_Data\'; % atlasroot = '\\atlas\carbonlab\Spray_processing\data\'; atlasroot = '\\atlas\projectlibrary\901805_Coastal_Biogeochemical_Sensing\Spray_Data\'; % ============== Get .sat file ======================== % loop through all active deployments for i = 1:length(activemissions) % extract mission name missionname = activemissions{i}; % get sat filename from deployment name satfname = sprintf('0%s.sat',missionname(4:6)); % directory where sat files are saved for this deployment satdirect = [depdirectory,missionname,'\sat_files\']; % log start of sat file download mlogfid = fopen(sprintf('%slog.txt',missionname), 'a'); fprintf(mlogfid, '%s,Started update_GliderVIZ.m\r\n',datestr(now)); fclose(mlogfid); % check to see if .sat file exists. If not, download it. 6/25/2019 YT needed for first dep. if(~isfile([satdirect,satfname])) % download .sat file from sftp server disp('No sat file found. Start downloading sat file...'); % this became obsolete when bog-fire server ubuntu was updated in late 2020. % sftp_conn = sftp_simple_get('bog-fire', 'gs', 'd4415aVis', satfname, satdirect, '/home/gs/data/active/'); % ssh2_close(sftp_conn); % close sftp structure % new approach: use putty to get data. % !!! This worked only if PuTTY verison was last updated on Sep 2019. The most recent one % (June 2020) returned error: % ssh_init: Network error: Cannot assign requested address % The version of PuTTY that works is saved in dropbox GliderVIZ, as well as Project Library. unix(sprintf('pscp -pw d4415aVis gs@bog-fire.shore.mbari.org:/home/gs/data/active/%s %s',satfname, satdirect)); % system(sprintf('pscp -pw d4415aVis gs@bog-fire.shore.mbari.org:/home/gs/data/active/%s %s',satfname, satdirect), '-echo'); disp('Finished downloading sat file'); end % copy current sat file to DepID_.sat; this will be used to see whether new data has come in % oldsatfname = [satfname(1:end-4),'_',datestr(now,'mmddHHMM'),'.sat']; % used this for debugging oldsatfname = [satfname(1:end-4),'_.sat']; copyfile([satdirect,satfname], [satdirect,oldsatfname]); disp('Finished copying current sat file'); % download .sat file from sftp server disp('Start downloading sat file...'); % sftp_conn = sftp_simple_get('bog-fire', 'gs', 'd4415aVis', satfname, satdirect, '/home/gs/data/active/'); % ssh2_close(sftp_conn); % close sftp structure % log to mission specific file mlogfid = fopen(sprintf('%slog.txt',missionname), 'a'); fprintf(mlogfid, '%s,Start downloading sat file\r\n',datestr(now)); fclose(mlogfid); unix(sprintf('pscp -pw d4415aVis gs@bog-fire.shore.mbari.org:/home/gs/data/active/%s %s',satfname, satdirect)); disp('Finished downloading sat file'); % compare size between the newly downloaded sat file satnew = dir([satdirect,satfname]); satold = dir([satdirect,oldsatfname]); % % obsolete section of code. was using to check size of file as it was downloaded. % logfid = fopen('log.txt', 'a'); % fprintf(logfid, '%s,%s,newbytes:%0.0f,oldbytes:%0.0f\r\n', datestr(now), missionname, satnew.bytes, satold.bytes); % fclose(logfid); % log to mission specific file mlogfid = fopen(sprintf('%slog.txt',missionname), 'a'); fprintf(mlogfid, '%s,Finished downloading sat file\r\n',datestr(now)); fclose(mlogfid); % save to atlas. 10/11/2020 YT % copyfile([satdirect,satfname], sprintf('%ssat_files\\raw_satfiles\\%s.sat',atlasroot,missionname)); % copyfile([satdirect,satfname], sprintf('%s%s/sat_files/',atlasroot,missionname)); %Obsolete because saving directly to Atlas - JSL % ============= IF SAT UPDATED, UPDATE GLIDERVIZ =================== if(satnew.bytes > satold.bytes || force_process) % check to see if the new file is bigger than the previous file. If so, parse. if not, ignore. % load cal file run([depdirectory,missionname,'\cal_',missionname,'.m']); % load calib coeff for this deployment disp('Loaded cal file. Start parsing sat file...'); satfpath = [satdirect,satfname]; % record in log mlogfid = fopen(sprintf('%slog.txt',missionname), 'a'); fprintf(mlogfid, '%s,New Sat file... Start parsing\r\n',datestr(now)); fclose(mlogfid); % parse sat file to matlab s = parse_spraysat2mat(satfpath, cal, missionname, atlasroot); % record in log mlogfid = fopen(sprintf('%slog.txt',missionname), 'a'); fprintf(mlogfid, '%s,Finished parsing sat file to mat\r\n',datestr(now)); fclose(mlogfid); % Save sat file in local folder (probably don't need this anymore) save('most_recent_sat.mat', 's'); % write in another log proffid = fopen('proflog.txt', 'a'); fprintf(proffid, '%s, %s, nprof=%0.0f\r\n', datestr(now), missionname, length(s.sdn)); fclose(proffid); disp('Finished parsing to matlab. Now writing Gliderviz file...'); % bandaid for 20703401. remove this if statement afterwards. Just delete the first 2 profiles % from 's' structure so it doesn't break GliderVIZ. Note that the profile name on the real time % emailed plots are off by 2 now. if(strcmp(missionname,'20703401')) vars = fieldnames(s); vars(ismember(vars, {'sdn_start', 'lat_start', 'lon_start', 'depID'})) = []; for v = 1:length(vars) s.(vars{v})(:,1:2) = []; end end proffid = fopen('proflog.txt', 'a'); fprintf(proffid, '%s, nprof=%0.0f\r\n', datestr(now),length(s.sdn)); fclose(proffid); % Write gliderviz file, and upload it to sirocco convert_sat2gliderviztxt(s, missionname, true); % copy gliderviz text files to atlas. 10/11/2020 YT % copyfile([missionname,'RT.txt'], sprintf('%ssat_files\\gliderviz\\',atlasroot)); % copyfile([missionname,'RT.cfg'], sprintf('%ssat_files\\gliderviz\\',atlasroot)); copyfile([missionname,'RT.txt'], sprintf('%s%s\\',atlasroot,missionname)); % decided to move it to Project Library for real time data instead; 12/1/2020 copyfile([missionname,'RT.cfg'], sprintf('%s%s\\',atlasroot,missionname)); % copy gliderviz text file to dropbox folder copyfile([missionname,'RT.txt'], [depdirectory,missionname,'\\']); copyfile([missionname,'RT.cfg'], [depdirectory,missionname,'\\']); % delete gliderviz files delete([missionname,'RT.txt']); delete([missionname,'RT.cfg']); % movefile([missionname,'RT.txt'], [depdirectory,missionname], 'f'); % movefile([missionname,'RT.cfg'], [depdirectory,missionname], 'f'); mlogfid = fopen(sprintf('%slog.txt',missionname), 'a'); fprintf(mlogfid, '%s,Finished writing to GliderVIZ\r\n',datestr(now)); fclose(mlogfid); % make summary plot plot_spraysat(s, {[atlasroot,missionname,'\sat_files\plots\']}); mlogfid = fopen(sprintf('%slog.txt',missionname), 'a'); fprintf(mlogfid, '%s,Finished summary plots\r\n',datestr(now)); fclose(mlogfid); % create spray map for 029, CANON if(strcmp(missionname,'20A02901')) nplot = 10; fname = 'times'; fsize = 12; f1 = figure; hold on; box on; m_proj('miller', 'lat', [nanmin(s.lat)-0.5 nanmax(s.lat)+0.5], 'lon', [nanmin(s.lon)-0.5 nanmax(s.lon)+0.5]); % set range for world map m_coast('color', [0 0 0]); % draw coast lines m_plot(s.lon(end-nplot:end), s.lat(end-nplot:end), 'ro', 'markerfacecolor', 'r', 'markersize', 4); % plot cruise tracks. m_plot(s.lon(end), s.lat(end), 'ko', 'markerfacecolor', 'k', 'markersize', 4); % plot last surface black set(gca, 'yticklabel', [], 'xticklabel', []); % erase x and y labels. xlabel('Lon');%, 'fontsize', fsize, 'fontname', fname); ylabel('Lat');%, 'fontsize', fsize, 'fontname', fname); try mapfname = '\\atlas\projectlibrary\901805_Coastal_Biogeochemical_Sensing\CANON_Fall_2020\Spray_location_map.jpg'; saveas(f1, mapfname); catch ME mlogfid = fopen('Errorlog.txt', 'a'); fprintf(mlogfid, '%s, %s, map plot filename: %s, Error: %s\r\n',datestr(now), missionname, mapfname, ME.identifier); fclose(mlogfid); end % pause(3); % movefile('Spray_location_map.jpg', 'Y:\901805-Coastal Biogeochemical Sensing\CANON_Fall_2020\', 'f'); close all; end mlogfid = fopen(sprintf('%slog.txt',missionname), 'a'); fprintf(mlogfid, '%s,Finished map plots\r\n',datestr(now)); fclose(mlogfid); % if(strcmp(missionname,'20A03401')) % % just for 20703401 % try % plot_c1_wind_contour(s, '\\atlas\projectlibrary\901805_Coastal_Biogeochemical_Sensing\CANON_Fall_2020\C1_contour_plots\'); % catch ME % mlogfid = fopen('Errorlog.txt', 'a'); % fprintf(mlogfid, '%s, %s, contour plot Error: %s\r\n',datestr(now), missionnamt ME.identifier); % fclose(mlogfid); % end % else % try % plot_c1_wind_contour(s, '\\atlas\projectlibrary\901805_Coastal_Biogeochemical_Sensing\CANON_Fall_2020\C1_M1_transect_contour_plots\'); % catch ME % mlogfid = fopen('Errorlog.txt', 'a'); % fprintf(mlogfid, '%s, %s, contour plot Error: %s\r\n',datestr(now), missionname, ME.identifier); % fclose(mlogfid); % end % end % mlogfid = fopen(sprintf('%slog.txt',missionname), 'a'); % fprintf(mlogfid, '%s,Finished c1 plots\r\n',datestr(now)); % fclose(mlogfid); % end disp(['GliderViz updated on ',datestr(now)]); end % clear variables % clearvars -except depdirectory activemissions if(strcmp(missionname,'21202901')) % extract 18340 text file from mbari ftp server % open ftp S = ftp('ftp.mbari.org'); % Passive Mode. This section of code provided by Josh Plant. cd(S); sf = struct(S); sf.jobject.enterLocalPassiveMode(); cd(S, 'pub/SOCCOM/FloatVizData/'); mget(S, '5906441.TXT', [atlasroot,missionname,'\']); % Download target file close(S); % Close ftp % convert floatviz text file to mat flt = fltviz2mat_V4([atlasroot,missionname,'\5906441.txt']); kmlwritepoint([atlasroot,missionname,'\19719'], flt.LAT, flt.LON, 'color', 'red'); end end if(~isempty(activemissions)) proffid = fopen('proflog.txt', 'a'); fprintf(proffid, '%s, Finished update_GliderVIZ_sat.m\r\n', datestr(now)); fclose(proffid); end % % exit;