%% Import data from text file.
% Script for importing data from the following text file:
%
%    \\atlas\projectlibrary\CPF\Data\CPF003\TankTests\2021Mar04\CPF003.001.2021-3-4T18-13-17.eng.txt
%
% To extend the code to different selected data or a different text file,
% generate a function instead of a script.

% Auto-generated by MATLAB on 2021/03/08 13:05:29

%% Initialize variables.
filename = '\\atlas\projectlibrary\CPF\Data\CPF003\TankTests\2021Mar04\CPF003.001.2021-3-4T18-13-17.eng.txt';
delimiter = '\t';

%% Read columns of data as text:
% For more information, see the TEXTSCAN documentation.
formatSpec = '%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%[^\n\r]';

%% Open the text file.
fileID = fopen(filename,'r');

%% Read columns of data according to the format.
% This call is based on the structure of the file used to generate this
% code. If an error occurs for a different file, try regenerating the code
% from the Import Tool.
dataArray = textscan(fileID, formatSpec, 'Delimiter', delimiter, 'TextType', 'string',  'ReturnOnError', false);

%% Close the text file.
fclose(fileID);

%% Convert the contents of columns containing numeric text to numbers.
% Replace non-numeric text with NaN.
raw = repmat({''},length(dataArray{1}),length(dataArray)-1);
for col=1:length(dataArray)-1
    raw(1:length(dataArray{col}),col) = mat2cell(dataArray{col}, ones(length(dataArray{col}), 1));
end
numericData = NaN(size(dataArray{1},1),size(dataArray,2));

for col=[20,23,24,25,26,29,30,40]
    % Converts text in the input cell array to numbers. Replaced non-numeric
    % text with NaN.
    rawData = dataArray{col};
    for row=1:size(rawData, 1)
        % Create a regular expression to detect and remove non-numeric prefixes and
        % suffixes.
        regexstr = '(?<prefix>.*?)(?<numbers>([-]*(\d+[\,]*)+[\.]{0,1}\d*[eEdD]{0,1}[-+]*\d*[i]{0,1})|([-]*(\d+[\,]*)*[\.]{1,1}\d+[eEdD]{0,1}[-+]*\d*[i]{0,1}))(?<suffix>.*)';
        try
            result = regexp(rawData(row), regexstr, 'names');
            numbers = result.numbers;
            
            % Detected commas in non-thousand locations.
            invalidThousandsSeparator = false;
            if numbers.contains(',')
                thousandsRegExp = '^[-/+]*\d+?(\,\d{3})*\.{0,1}\d*$';
                if isempty(regexp(numbers, thousandsRegExp, 'once'))
                    numbers = NaN;
                    invalidThousandsSeparator = true;
                end
            end
            % Convert numeric text to numbers.
            if ~invalidThousandsSeparator
                numbers = textscan(char(strrep(numbers, ',', '')), '%f');
                numericData(row, col) = numbers{1};
                raw{row, col} = numbers{1};
            end
        catch
            raw{row, col} = rawData{row};
        end
    end
end

% Convert the contents of columns with dates to MATLAB datetimes using the
% specified date format.
try
    dates{2} = datetime(dataArray{2}, 'Format', 'dd-MMM-yyyy', 'InputFormat', 'dd-MMM-yyyy');
catch
    try
        % Handle dates surrounded by quotes
        dataArray{2} = cellfun(@(x) x(2:end-1), dataArray{2}, 'UniformOutput', false);
        dates{2} = datetime(dataArray{2}, 'Format', 'dd-MMM-yyyy', 'InputFormat', 'dd-MMM-yyyy');
    catch
        dates{2} = repmat(datetime([NaN NaN NaN]), size(dataArray{2}));
    end
end

dates = dates(:,2);

%% Split data into numeric and string columns.
rawNumericColumns = raw(:, [20,23,24,25,26,29,30,40]);
rawStringColumns = string(raw(:, [1,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,21,22,27,28,31,32,33,34,35,36,37,38,39,41,42,43,44,45]));


%% Replace non-numeric cells with NaN
R = cellfun(@(x) ~isnumeric(x) && ~islogical(x),rawNumericColumns); % Find non-numeric cells
rawNumericColumns(R) = {NaN}; % Replace non-numeric cells

%% Make sure any text containing <undefined> is properly converted to an <undefined> categorical
for catIdx = [1,3,4]
    idx = (rawStringColumns(:, catIdx) == "<undefined>");
    rawStringColumns(idx, catIdx) = "";
end

%% Create output variable
CPF003 = table;
CPF003.VarName1 = categorical(rawStringColumns(:, 1));
CPF003.Date = dates{:, 1};
CPF003.Time = rawStringColumns(:, 2);
CPF003.State = categorical(rawStringColumns(:, 3));
CPF003.Comment = categorical(rawStringColumns(:, 4));
CPF003.Pressure = rawStringColumns(:, 5);
CPF003.BelPos = rawStringColumns(:, 6);
CPF003.PumpVolts = rawStringColumns(:, 7);
CPF003.PumpmilliAmps = rawStringColumns(:, 8);
CPF003.PumpJoules = rawStringColumns(:, 9);
CPF003.HotelVolts = rawStringColumns(:, 10);
CPF003.HotelmilliAmps = rawStringColumns(:, 11);
CPF003.HotelJoules = rawStringColumns(:, 12);
CPF003.CTDTemp = rawStringColumns(:, 13);
CPF003.CTDSal = rawStringColumns(:, 14);
CPF003.RawPress = rawStringColumns(:, 15);
CPF003.LPFPress = rawStringColumns(:, 16);
CPF003.LPFVel = rawStringColumns(:, 17);
CPF003.DiffVel = rawStringColumns(:, 18);
CPF003.Diff2Vel = cell2mat(rawNumericColumns(:, 1));
CPF003.SlopeofLPFPress = rawStringColumns(:, 19);
CPF003.CTDSamplePeriod = rawStringColumns(:, 20);
CPF003.LPS331Press = cell2mat(rawNumericColumns(:, 2));
CPF003.LPS311Temp = cell2mat(rawNumericColumns(:, 3));
CPF003.SHT21Humid = cell2mat(rawNumericColumns(:, 4));
CPF003.SHT21Temp = cell2mat(rawNumericColumns(:, 5));
CPF003.PIDSP = rawStringColumns(:, 21);
CPF003.PIDPV = rawStringColumns(:, 22);
CPF003.PIDY1 = cell2mat(rawNumericColumns(:, 6));
CPF003.PIDY2 = cell2mat(rawNumericColumns(:, 7));
CPF003.PIDI = rawStringColumns(:, 23);
CPF003.PIDv = rawStringColumns(:, 24);
CPF003.PIDu = rawStringColumns(:, 25);
CPF003.PIDuCPS = rawStringColumns(:, 26);
CPF003.PIDh = rawStringColumns(:, 27);
CPF003.FLBB = rawStringColumns(:, 28);
CPF003.Topt = rawStringColumns(:, 29);
CPF003.Tphase = rawStringColumns(:, 30);
CPF003.Rphase = rawStringColumns(:, 31);
CPF003.NO3 = cell2mat(rawNumericColumns(:, 8));
CPF003.pHVrs = rawStringColumns(:, 32);
CPF003.OCR = rawStringColumns(:, 33);
CPF003.GPS = rawStringColumns(:, 34);
CPF003.IridModem = rawStringColumns(:, 35);
CPF003.Msc = rawStringColumns(:, 36);

% For code requiring serial dates (datenum) instead of datetime, uncomment
% the following line(s) below to return the imported dates as datenum(s).

% CPF003.Date=datenum(CPF003.Date);

%% Clear temporary variables
clearvars filename delimiter formatSpec fileID dataArray ans raw col numericData rawData row regexstr result numbers invalidThousandsSeparator thousandsRegExp dates blankDates anyBlankDates invalidDates anyInvalidDates rawNumericColumns rawStringColumns R catIdx idx;