function out = merge_discrete_pH(DataDir, SampleFilter, plot_it)

%This function parses the LabView generated discrete colorimetric pH data
%files and merges them producing an nx2 time series of pH Total
%measurements
%
% out = merge_discrete_pH(DataDir, SampleFilter, plot_it)
% INPUTS:
%    DataDir      = path to directory where data files are stored
%    SampleFilter = case insensitive string used to filterr data by sample
%                   ID, for example 'K0' would get all pH used for K0
%                   calibrations. If no filter use ''
%    plot_it      = 1 means make a plot
%
% OUPUT:
%    out  = n x 3 array [Matlab SDN, pH Total, pH QC]
%
% EXAMPLES:
%    fp   = '\\atlas\chem\DuraFET\K0 for DSD Stems\SpecpH_data\'
%    data = merge_discrete_pH(fp, 'K0',1)
%    data = merge_discrete_pH(fp, '', 1)
%
% CHANGE LOG:
% 12/14/2022 - JP Updated date number extraction to check for non date-strings
%    strings and to check for two different time formats HH:MM & HH:MM:SS



% *************************************************************************
%TESTING
% % DataDir = ['C:\Users\jplant\Documents\MATLAB\Apps\pH_Calibration\', ...
% %            'test data\discrete pH\'];
% % DataDir ='\\atlas\chem\DuraFET\K0 for DSD Stems\SpecpH_data\';
% DataDir ='\\atlas\chem\DuraFET\CALIBRATION\SpecpH_data\';
% SampleFilter = 'K0';
% 
% % SampleFilter = '';
% plot_it = 1;
% *************************************************************************


% GET LISTING OF LABVIEW GENERATED FILES
tmp = dir([DataDir,'SOCCOM*']);
fnames = {tmp.name}';
clear tmp


% STEP THROUGH FILES & EXTRACT DATA
out = ones(2000,2) *NaN;
line_ct = 1;
for fct = 1: size(fnames,1)
    fn = fnames{fct};
    fp = [DataDir, fn];
    disp(fp)
    fid = fopen(fp);
    
    % LOOK FOR HEADER LINE
    tline = '';
    while ischar(tline)
        if regexp(tline,'^Comments','once')
            break % Header line found
        end
        tline = fgetl(fid);
    end
    
    if tline == -1 % NO HEADER ROW FOUND
        disp('NO HEADER LINE FOUND - USING DEFAULT HEADER FORMAT');
        frewind(fid)
        fmt = repmat('%*s',1,24); % 24 columns
        % need to start at end first & work to begining
        fmt = regexprep(fmt, '%\*s', '%s', 24); % measur3ed pH;
        fmt = regexprep(fmt, '%\*s', '%s', 5); % task identifier i.e. "K0"
        fmt = regexprep(fmt, '%\*s', '%s', 2); % date
    else
        hdr = regexp(tline,'\t','split');
        fmt = '';
        for i = 1:size(hdr,2)
            if regexp(hdr{i},'DateTime|SampID|pH_tot','once')
                fmt = [fmt,'%s'];
            else
                fmt = [fmt,'%*s'];
            end
        end
    end
    
    d = textscan(fid, fmt, 'Delimiter', '\t'); % parse the data!
    
    %Filter the data
    % THE TIME FORMAT IS INCONSISTENT!!! HH:MM or HH:MM:SS jp 12/14/2022
    % TIME COLUMN ALSO HAS NON TIME STRINGS
    dcell = d{1,1}; % cell of date strings
    tg = ~cellfun(@isempty, regexp(dcell,'^\d+','once')); % test for string starting with number
    if sum(~tg) > 0
        fprintf('WARNING: NON DATE STRING FORMAT DETECTED in %s\n & WILL BE IGNORED\n', fp);
    end


    tHMS  = ~cellfun(@isempty, regexp(dcell,'\d+:\d+:\d+','once')); % test for HH:MM:SS
    sdn   = ones(size(dcell))*NaN; % predim serial date number array
    if sum(tHMS) > 0
        fprintf('WARNING: NON STANDARD DATE FORMAT DETECTED (HH:MM:SS) in %s\n', fp);
        sdn(tHMS&tg) = datenum(dcell(tHMS&tg),'mm/dd/yyyy HH:MM:SS PM');
    end
    sdn(~tHMS&tg) = datenum(dcell(~tHMS&tg),'mm/dd/yyyy HH:MM PM');


    if ~isempty(SampleFilter)
        t1 = strcmpi(d{1,2}, SampleFilter); 
        if sum(t1) == 0
            disp(['No data found matching input filter: ',SampleFilter])
            out = [];
            continue
        end
        tmp = [sdn(t1), str2double(d{1,3}(t1))];
    else
        tmp = [sdn, str2double(d{1,3})];        
    end
    
    rtmp = size(tmp,1);
    out(line_ct:line_ct+rtmp-1,:) = tmp;
    line_ct = line_ct +rtmp;
end
out    = out(1:line_ct-1,:);
if ~isempty(out)
    [~,ia] = sort(out(:,1)); % Just to make sure
    out    = out(ia,:);
    out    = [out, out(:,1)*0]; % add quality flag - assume all good to start
    
    if plot_it == 1
        TT = datetime(out(:,1),'ConvertFrom','datenum');
        plot(TT,out(:,2),'bo')
        if isempty(SampleFilter)
            tstr = ['Discrete colorimetric pH data', ...
                '    N = ', num2str(size(out,1))];
        else
            tstr = ['Discrete colorimetric pH data     ', ...
                'SampleFilter = "', SampleFilter,'"', ...
                '    N = ', num2str(size(out,1))];
        end
        title(tstr)
        ylabel('pH Total')
    end
end
                
                
        

        
    