function output=charge_VARS(ROVdive,liste_concepts,varargin)

% CHARGE_VARS : récupère des données à partir de VARS et d'une query SQL. 
% Renvoie [] s'il y a moins de 3 standard depths pour le ROVdive, sinon un obj_intTS avec 0 si aucune annotation n'est trouvée, NaN pour les depths non sampled.
% Note que ROVdive est en référence au numéro dans la midwater table, qui peut être faux (on ne query pas dessus, c'est juste pour référence).
%
% output=charge_VARS(ROVdive,liste_concepts,varargin)
%
% 'outputdiff' (implique speciesRob) 	renvoie la différence entre cette fonction et les données de Rob, au lieu des données
% 'speciesRob'									renvoie une table avec les mêmes espèces que la table de Rob (au lieu du défaut présent dans VARS)
%
% Monique, octobre 2014, MBARI

[~,flag]=read_varargin(varargin,{},{'outputdiff','speciesRob'});
flag.correctdepth=true;


% -------------- input

if nargin<2, liste_concepts={}; else if ~iscell(liste_concepts), liste_concepts={liste_concepts}; end, liste_concepts=reshape_ligne(liste_concepts); end
liste_depths=reshape_col([50 100:100:1000]); 
if flag.speciesRob || flag.outputdiff, data_transects=charge_transectsMidW('counts3D','transect'); liste_concepts=data_transects.species; end

liste_ROV={'Ventana','Doc Ricketts','Tiburon'}; irov=1;
while isempty(strfind(ROVdive,liste_ROV{irov})), irov=irov+1; end
ROV=liste_ROV{irov}; divenumber=str2num(ROVdive(length(ROV)+1:end));


% -------------- query the database

% find times for each depth and keep only standard depths
v=sqlquery(['SELECT id, depth_m, start_datetime_gmt, end_datetime_gmt, volume_m3, day_night FROM transect_data WHERE dive = ',num2str(divenumber),' AND rov = ''',ROV,''''],'midwater');
if isempty(v.depth_m), disp([ROVdive,' ~ No data in VARS?']), output=[]; return, end
iok = eqtot(v.depth_m,liste_depths) & eqStrtot(v.day_night,'D'); 		% ajout filtre sur D à cause du "dive" Ventana3750 qui en contient 2 en fait
for varname=permute(fieldnames(v),[2 1]), varname=varname{:}; v.(varname)=v.(varname)(iok); end
depth_sampled=v.depth_m; 
if isempty(depth_sampled), disp([ROVdive,' ~ Not a day dive or not standard depths']), output=[]; return, end
if length(depth_sampled)<3 && length(liste_depths)>3, disp([ROVdive,' ~ Less than 3 standard depths sampled - not loaded']), output=[]; return, end

% sql based on ROV & concepts
sql=['SELECT ConceptName,DEPTH,RecordedDate,ObservationID_FK,LinkName,LinkValue,VideoArchiveName,DiveNumber FROM Annotations ' ...
	'WHERE (RovName = ''',ROV,''') AND ('];
if ~isempty(liste_concepts), liste_concepts_sqlstr='';
	for concept=liste_concepts, liste_concepts_sqlstr=[liste_concepts_sqlstr,'ConceptName = ''',concept{:},''' OR ']; end
	sql=[sql,liste_concepts_sqlstr(1:end-4),') AND (']; 
end
for idepth=1:length(depth_sampled), 
	sql=[sql,'((RecordedDate >= ''',datestr(v.start_datetime_gmt(idepth),'yyyy-mm-dd HH:MM:SS'),''') AND (RecordedDate <= ''',datestr(v.end_datetime_gmt(idepth),'yyyy-mm-dd HH:MM:SS'),''')) OR '];
end
sql=[sql(1:end-4),')'];


% query all (légèrement plus rapide qu'en loop) et ensuite changement des depths
output_ini=sqlquery(sql,'VARS'); if trie_vecteur(output_ini.DiveNumber)~=divenumber, disp(['DiveNumber in VARS = ',num2str(trie_vecteur(output_ini.DiveNumber))]), end
ibad=logical(ones(length(output_ini.DEPTH),1)*0); depth_ini=output_ini.DEPTH; 
for idepth=1:length(depth_sampled), itime=output_ini.RecordedDate>=v.start_datetime_gmt(idepth) & output_ini.RecordedDate<=v.end_datetime_gmt(idepth);
	ibad (itime & abs(output_ini.DEPTH-depth_sampled(idepth))>40 ) = true;		% filtre pour repérer les depths aberrantes à cause d'un mauvais timestamp par ex
	output_ini.DEPTH(itime)=depth_sampled(idepth); 
end
if max(ibad), disp(['Annotations supprimées pour cause de depths foireuses (',ROVdive,') : ',num2str(sum(ibad))])
	if 0		% affichage des profils foireux
		aa = sqlquery(['SELECT RecordedDate,DEPTH From Annotations WHERE (RovName = ''',ROV,''') AND (DiveNumber = ',num2str(divenumber),')'],'VARS'); 
		ii=aa.RecordedDate>datenum(1990,1,1);
		affiche_series(trie_matrice([aa.RecordedDate(ii),aa.DEPTH(ii)]),[],'yreverse','titre',[ROV,num2str(divenumber),' ~ ',num2str(sum(ibad))],'xlabel','time','ylabel','depth'), hold on, global style_global
		for idepth=1:length(depth_sampled), 
			plot([v.start_datetime_gmt(idepth) v.start_datetime_gmt(idepth)],get(gca,'YLim'),'Color',couleur2rgb(style_global.couleurs{idepth+1}),'LineWidth',2), 
			plot([v.end_datetime_gmt(idepth) v.end_datetime_gmt(idepth)],get(gca,'YLim'),'Color',couleur2rgb(style_global.couleurs{idepth+1}),'LineWidth',2), 
			plot([v.start_datetime_gmt(idepth) v.end_datetime_gmt(idepth)],[v.depth_m(idepth) v.depth_m(idepth)],'Color',couleur2rgb(style_global.couleurs{idepth+1}),'LineWidth',2)
		end
		scatter(output_ini.RecordedDate(ibad),depth_ini(ibad),30,'r')
		ROVdivestr=ROVdive; ROVdivestr(strfind(ROVdivestr,' '))='';
		print_fig(ROVdivestr,'/home/monique/mbarinet/home_MBARI/dives/')
	end
end
if flag.correctdepth, for varname=permute(fieldnames(output_ini),[2 1]), varname=varname{:}; output_ini.(varname)=output_ini.(varname)(~ibad); end, end

% no data found
if isempty(output_ini.DEPTH) 
	if isempty(liste_concepts), 
		sql=['SELECT RecordedDate FROM Annotations WHERE (RovName = ''',ROV,''') ' ...
			'AND (RecordedDate >= ''',datestr(min(v.start_datetime_gmt),'yyyy-mm-dd HH:MM:SS'),''') AND (RecordedDate <= ''',datestr(max(v.end_datetime_gmt),'yyyy-mm-dd HH:MM:SS'),''')'];
		output_ini = sqlquery(sql,'VARS');
		datestr(minmax(output_ini.RecordedDate))
		datestr(minmax([v.start_datetime_gmt;v.end_datetime_gmt]))
		disp(ROVdive)
		error(['no data? id = ',num2str(min(v.id)),' to ',num2str(max(v.id))])
	else 
		if flag.outputdiff, output=sum(aplat(data_transects.all(:,:,eqStrtot(data_transects.attributs.ROVdive,ROVdive))));
		else 
			output=obj_intTS(liste_concepts,'depth',liste_depths,'time',floor(min(v.start_datetime_gmt)),'datablank'); 
			output.all(:)=0; output.all(:,~eqtot(output.depth,depth_sampled))=NaN; output.attributs.ROVdive=ROVdive; 
		end
		return
	end
end


% -------------- clean up query by removing duplicates

% finds annotations tagged with identity-reference
tocheck=find(eqStrtot(output_ini.LinkName,'identity-reference'));

if ~isempty(tocheck)
% construct uniqueID for annotations to check, based on referenceID, concept name and video archive name
	uniqueID=output_ini.ObservationID_FK*NaN;
	for icheck=reshape_ligne(tocheck), VideoArchiveName=output_ini.VideoArchiveName{icheck}; iHD=strfind(VideoArchiveName,'HD'); VideoArchiveName(iHD:iHD+1)='';
		s = java.lang.String([output_ini.ConceptName{icheck},VideoArchiveName,output_ini.LinkValue{icheck}]); uniqueID(icheck)=s.hashCode(); 
	end

% for each unique ID, finds the corresponding ObservationsID_FK and replace them by an unique number
	for id=permute(unique(uniqueID(~isnan(uniqueID))),[2 1]), iid = uniqueID==id;	
		observationsID=unique(output_ini.ObservationID_FK(iid));
		output_ini.ObservationID_FK(eqtot(output_ini.ObservationID_FK,observationsID))=min(observationsID);	
	end
end

% clean up duplicate ObservationsID_FK (both due to processing above or other duplicates)
[~,iok]=unique(output_ini.ObservationID_FK); for varname=permute(fieldnames(output_ini),[2 1]), varname=varname{:}; output_ini.(varname)=output_ini.(varname)(iok); end


% -------------- get the counts at each depth


liste_ConceptNameQuery=trie_vecteur(output_ini.ConceptName); liste_DepthQuery=trie_vecteur(output_ini.DEPTH);
if isempty(liste_concepts), liste_concepts=liste_ConceptNameQuery; end
if isempty(liste_depths), liste_depths=liste_DepthQuery; end
output_final=obj_intTS(liste_concepts,'depth',liste_depths,'time',floor(min(v.start_datetime_gmt)),'datablank'); output_final.attributs.ROVdive=ROVdive;
if ~isempty(liste_ConceptNameQuery(~eqStrtot(liste_ConceptNameQuery,output_final.species))), disp([num2str(divenumber),' ~ ConceptName non rentrés ?']), end
liste_concepts=liste_ConceptNameQuery(eqStrtot(liste_ConceptNameQuery,output_final.species));

output_ini.idConcept=output_ini.DEPTH*NaN; 
for iconcept=reshape_ligne(find(eqStrtot(output_final.species,liste_ConceptNameQuery))), output_ini.idConcept(eqStrtot(output_ini.ConceptName,output_final.species{iconcept}))=iconcept; end
if length(liste_depths)==1, liste_bornesdepths=[liste_depths-10 liste_depths+10];
else dz=(liste_depths(2:end)-liste_depths(1:end-1))/2; liste_bornesdepths=[liste_depths(1)-dz(1);liste_depths(1:end-1)+dz;liste_depths(end)+dz(end)];
end

[output_final.all,~,mid] = histcn([output_ini.idConcept,output_ini.DEPTH], 0.5:length(output_final.species)+0.5, liste_bornesdepths); 
output_final.all(isnan(output_final.all))=0;
output_final.all(:,~eqtot(output_final.depth,depth_sampled))=NaN;		% depths that were not sampled

if flag.outputdiff
	mat_transects=data_transects.all(eqStrtot(data_transects.species,output_final.species),eqtot(data_transects.depth,output_final.depth),eqStrtot(data_transects.attributs.ROVdive,ROVdive));
	difference=abs((mat_transects-output_final.all));
	diff_max=max(aplat(difference));
	if diff_max>0, [ispecies,idepth]=find(difference==diff_max); output_final.species{ispecies(1)}, output_final.depth(idepth(1)), end
	output=diff_max;
else
	output=output_final;
end



return



%{
% Search for wrong dives (overlapping transects, depths too far from the theoretical depth (flag à mettre à 1 pour l'affichage de profils foireux))

v=sqlquery('SELECT id, start_datetime_gmt, end_datetime_gmt, rov, dive FROM transect_data','midwater');
ibad_all=[];
for iid=1:length(v.id), 
	ibad1=find(v.start_datetime_gmt<=v.start_datetime_gmt(iid) & v.end_datetime_gmt>=v.start_datetime_gmt(iid)); ibad1=ibad1(ibad1~=iid);
	ibad2=find(v.start_datetime_gmt<=v.end_datetime_gmt(iid) & v.end_datetime_gmt>=v.end_datetime_gmt(iid)); ibad2=ibad2(ibad2~=iid);
	if ~isempty(ibad1) || ~isempty(ibad2), ibad_all=[ibad_all,v.id(iid)]; end
end
ROVdive_bad={};
for ibad=ibad_all, ROVdive_bad=[ROVdive_bad,{[v.rov{v.id==ibad},num2str(v.dive(v.id==ibad))]}]; disp([ROVdive_bad{end},' ',datestr(v.start_datetime_gmt(v.id==ibad))]), end
ROVdive_bad=trie_vecteur(ROVdive_bad);
disp(' ')
disp(' ')

v=sqlquery('SELECT DISTINCT rov, dive, day_night, year FROM transect_data','midwater'); 
iok = eqStrtot(v.day_night,'D') & v.year>=1997; for varname=permute(fieldnames(v),[2 1]), varname=varname{:}; v.(varname)=v.(varname)(iok); end
for idive=1:length(v.dive), ROVdive=[v.rov{idive},num2str(v.dive(idive))];
	if ~eqStrtot(ROVdive,ROVdive_bad)
		try output=charge_VARS(ROVdive,{},'outputdiff'); if output>0, disp([num2str(output),' ~ ',ROVdive]), disp('-----------'), end
		catch disp(['loading failed! ',ROVdive]), disp('-----------'), end
	end
end

%}






