readFlukeFile(filename, meastype, varargin)

%readFlukeFile - Read a file with recorded measurements from a Fluke multimeter
%
%  [DATA TIME] = readFlukeFile(FILENAME), reads a CSV file with recorded
%  measurements from a Fluke multimeter. It returns DATA (a vector 
%  of measurement values) and TIME (a vector of measurement times). The
%  "time" of each measurement is the elapsed time in seconds since the
%  start of the recording session. 
%
%    The expected workflow is the following:
%    1. Configure the Fluke(TM) multimeter to record a series of 
%       measurements
%    2. Use FlukeView Forms(TM) to download the measurements, and export
%       them to a CSV file
%    3. Use readFlukeFile to read the CSV file
%
%  [DATA TIME INFO] = readFlukeFile(FILENAME), also returns a structure
%  INFO with additional information about the measurements (such as
%  the units of the measurements and the start time of the recording
%  session). 
%
%  [DATA TIME INFO] = readFlukeFile(FILENAME, MEASTYPE), only returns
%  measurements of a particular type. Valid values of MEASTYPE are:
%     'Min'     - Average value within each measurement interval 
%     'Max'     - Maximum value within each measurement interval
%     'Average' - Average value within each measurement interval
%     'Sample'  - The actual measurement taken in that interval
%   By default, MEASTYPE is 'Sample'
%
%  [DATA TIME INFO] = readFlukeFile(FILENAME, MEASTYPE, 'AllowDuplicateStartTimes') 
%  returns all measurements of a particular type, including measurements that 
%  have the same start time. If 'AllowDuplicateStartTimes' is omitted, 
%  only a single measurement is returned for each unique start time.
%
%  [HDR1 HDR2 RAWDATA] = readFlukeFile(FILENAME, 'Raw'), returns the raw 
%  contents of FILENAME. HDR1 and HDR2 are structs with information about
%  the recording session. RAWDATA is an array of structs (one for each
%  measurement). 'Raw' cannot be used with 'AllowDuplicateStartTimes'.
%
%  Examples: 
%     
%     [data time] = readFlukeFile('sampledata.csv'); 
%     plot(time, data);
%     xlabel('time in seconds');
%     
%     [data time info] = readFlukeFile('sampledata.csv', 'Average'); 
%     plot(time, data);
%     xlabel('time in seconds');
%     ylabel(info.MeasurementUnits);
%
%     % allow duplicate start times in the data. Note that you can use a shortened
%     % version of the parameter (i.e., 'allowdup' instead of 'AllowDuplicateStartTimes')
%     [data time info] = readFlukeFile('TemperatureChange.csv','sample','AllowDuplicateStartTimes');
%     [data time info] = readFlukeFile('TemperatureChange.csv','sample','allowdup');
%  Note:
%   1) This function assumes that the CSV file was created using 
%      FlukeView Forms [http://www.fluke.com/flukeviewforms/]
%   2) This function was tested using output from the Fluke 289 multimeter
%      [http://www.fluke.com/289]. If you are using a different Fluke 
%      multimeter, you may want to verify the output (e.g., by comparing 
%      it to the measurements displayed in FlukeView Forms).

% File History
% * Sep-7-2011: Updated read_csv_file to handle multiple empty fields at 
%   end of header rows. Also added 'AllowDuplicateStartTimes' flag.
%
%  Copyright 2009-2011 The MathWorks, Inc.

function [out1, out2, out3] = readFlukeFile(filename, meastype, varargin)

if nargin < 1
    fprintf('Expecting the name of a CSV file. For example:\n');
    fprintf('  [data, time] = readFlukeFile(''sampledata.csv'');\n');
    fprintf('Type ''help readFlukeFile'' for more information\n');
    return;
end

% check & validate meastype
if ~exist('meastype', 'var')
    meastype = 'Sample';
end

validMeasTypes = {'Sample', 'Max', 'Average', 'Min', 'Raw'};
index = find(strcmpi(meastype, validMeasTypes));
if isempty(index)
    error('readFlukeFile:UnknownParameter', ...
        'MeasurementType should be one of: ''Sample'', ''Max'', ''Average'', ''Min'', or ''Raw''\n');
else
    meastype = validMeasTypes{index};
end

% check & validate allowDuplicateStartTimes
allowDuplicateStartTimes = false;
for i=1:numel(varargin)
    s = varargin{i};
    if ischar(s) && strncmpi(s, 'AllowDuplicateStartTimes', length(s))
        allowDuplicateStartTimes = true;
    else
        error('readFlukefile:UnknownParameter', ...
            'Only allowed parameter is ''AllowDuplicateStartTimes''\n');
    end
    if strcmpi(meastype, 'Raw')
         error('readFlukefile:InvalidParameter', ...
            '''Raw'' and ''AllowDuplicateStartTimes'' cannot be used together\n'); 
    end
end


[hdr1, hdr2, data] = read_csv_file(filename);

if strcmpi(meastype, 'Raw')
    out1 = hdr1; out2 = hdr2; out3 = data;
    return;
end

% We need condense the raw data into 
% 1) a vector of measured values
% 2) a vector of time elapsed since start of session
% 3) a structure with some high-level info

out1 = []; out2 = [];
out3.SessionStartTime = datestr(hdr1.Start_Time);
out3.SessionDuration = hdr1.Elapsed_Time;
    
% Filter out all the invalid measurements

% -- Only retain records with description "Interval"
intervalIndices = strcmpi('Interval', {data.Description});
data = data(intervalIndices);
if numel(data) == 0
    warning('readFlukeFile:NoData', 'Cannot find any ''Interval'' measurements');
    return;
end

% -- Only retain records with non-nan sample values
sampleValues = arrayfun(@(x) x.(meastype).value, data);
data = data(~isnan(sampleValues));
if numel(data) == 0
    warning('readFlukeFile:NoData', 'Cannot find valid ''Interval'' measurements');
    return;
end

% -- Ensure consistent and uniform units
% Sometimes the units can change (e.g., 'AC' instead of 'mV AC'). It is 
% difficult to recover the units multipler from the the units label, and
% better not to guess. Find out which units are used most often, keep
% records that use those units, and discard the rest.
allUnits = arrayfun(@(x) x.(meastype).units, data, 'Unif', false);
[uniqueUnits,ignore,occurrence] = unique(allUnits);
if numel(uniqueUnits) > 1
    numOccurrences = accumarray(occurrence(:), 1);
    fprintf('Multiple units found\n');
    for i=1:numel(uniqueUnits)
        fprintf('  %3d occurrences of ''%s'' \n', numOccurrences(i), uniqueUnits{i});
    end    
    [ignore,index] = max(numOccurrences);
    fprintf('Keeping records with ''%s'' and ignoring the rest\n', uniqueUnits{index});
    data = data(occurrence == index);
    defaultUnits = uniqueUnits{index};
else
    defaultUnits = uniqueUnits{1};
end
% data should always be non-empty at this point, since we
% are keeping the readings with units-used-most-often
assert(numel(data) ~= 0);

% -- Check if we need to remove readings with duplicate start times

sampleTimes = zeros(numel(data),1);
globalStartTime = hdr1.Start_Time;
for i=1:numel(data)
    sampleTimes(i) = etime(data(i).Start_Time, globalStartTime);
end

if ~allowDuplicateStartTimes
    % sometimes there are multiple records with the same start time
    % only retain the last of these multiple records    
    [uniqueSampleTimes, uniqueSampleIndices] = unique(sampleTimes, 'last');
    data = data(uniqueSampleIndices);
    sampleTimes = uniqueSampleTimes;
end

% extract the data values
sampleValues = arrayfun(@(x) x.(meastype).value, data);

out1 = sampleValues;
out2 = sampleTimes;
out3.MeasurementUnits = defaultUnits;    

end


%% -------------------------------------------
% READ_CSV_FILE
%  [hdr1,hdr2,data] = read_csv_file(filename)
%   Parses the CSV file and returns all the data in three  structures.
%
%   hdr1 is the first header data in the file  (lines 1 & 2 in the  file)
%   hdr2 is the second header data in the file (lines 3 & 4 in the  file)
%   data is a structure array (all the measurement records in the file)
%
function [hdr1,hdr2,data] = read_csv_file(filename)

fid = fopen(filename,'r');
if fid < 0
    error('readFlukeFile:InvalidFile', 'Unable to open %s', filename);
end

% use fgets(fid) rather than fid so that each textscan is guaranteed
% to begin at the start of the line

try
    header1 = textscan(fgets(fid), '%s%s%s%s%s%s%s', 1, 'Delimiter',',');
    val1 = textscan(fgets(fid),'%s%s%s%d%d%d%s', 1, 'Delimiter',',');
    
    header2 = textscan(fgets(fid), '%s%s%s%s%s', 1, 'Delimiter',',');
    val2 = textscan(fgets(fid),'%s%s%s%s%s',1, 'Delimiter',',');
    
    header3 = textscan(fgets(fid), '%s%s%s%s%s%s%s%s%s%s%s', 1, 'Delimiter',',');
    % note use of fid rather than fgets(fid); we want to read till EOF
catch ME
     error('readFlukeFile:InvalidFormat', 'Header rows (rows 1-5) of %s have an unexpected format', filename);
end

val3 = textscan(fid, '%d%s%s%s%s%s%s%s%s%s%s', 'Delimiter',',');
fclose(fid);

group1_field_parsers = struct( ...
    'Start_Time',      @parse_time, ...
    'Elapsed_Time',    @parse_duration, ...
    'Interval',        @parse_duration, ...
    'Total_readings',  @noop, ...
    'Intervals',       @noop, ...
    'Input_Events',    @noop, ...
    'Stop_Time',       @parse_time ...
    );

group2_field_parsers = struct( ...
    'Max_Time',    @parse_time, ...
    'Max',         @parse_measurement, ...
    'Average',     @parse_measurement, ...
    'Min',         @parse_measurement, ...
    'Min_Time',    @parse_time ...  
    );

group3_field_parsers = struct( ...
    'Reading',     @noop,   ...
    'Sample',      @parse_measurement, ...
    'Start_Time',  @parse_time, ...
    'Duration',    @parse_duration, ...
    'Max_Time',    @parse_time, ...
    'Max',         @parse_measurement, ...
    'Average',     @parse_measurement, ...
    'Min_Time',    @parse_time, ...
    'Min',         @parse_measurement, ...
    'Description', @noop,  ...
    'Stop_Time',   @parse_time ...
    );

try
    headerfields = getFieldNames(header1, group1_field_parsers);
    hdr1 = makeStruct(headerfields, val1, group1_field_parsers);
catch %#ok<CTCH>
    error('readFlukeFile:InvalidFormat', 'Lines 1 and 2 of ''%s'' have an unexpected format', filename);
end

try
    headerfields = getFieldNames(header2, group2_field_parsers);
    hdr2 = makeStruct(headerfields, val2, group2_field_parsers);
catch %#ok<CTCH>
    error('readFlukeFile:InvalidFormat', 'Lines 3 and 4 of ''%s'' have an unexpected format', filename);
end

try
    headerfields = getFieldNames(header3, group3_field_parsers);
    data = makeStruct(headerfields, val3, group3_field_parsers);
catch  %#ok<CTCH>
    error('readFlukeFile:InvalidFormat', 'Problem reading measurement data from ''%s''', filename);
end

end

%%
function actualFieldNames = getFieldNames(headers, fieldParsers)
actualFieldNames = cell(numel(headers),1);
for i=1:numel(headers)
    actualFieldNames{i} = strrep(headers{i}{1},' ','_');
end
% check that the fieldnames match what's expected
expectedFieldNames = fieldnames(fieldParsers);
if numel(actualFieldNames) ~= numel(expectedFieldNames)
    error('readFlukeFile:InvalidFields', 'Unexpected fields');
end
for i=1:numel(actualFieldNames)
    if ~strcmp(actualFieldNames{i}, expectedFieldNames{i})
         error('readFlukeFile:InvalidFields', 'Unexpected fields');
    end
end
end


%%
function s = makeStruct(headerfields, values, fieldParsers)
structinfo = cell(2,numel(values));
for i=1:numel(values)
    if isnumeric(values{i})
        vals = num2cell(values{i});
    else
        vals = values{i};
    end
    structinfo{1,i} = headerfields{i};
    parserFcn = fieldParsers.(headerfields{i});
    vals2 = parserFcn(vals);
    structinfo{2,i} = vals2(:);
end
s = struct(structinfo{:});
end

%%
function out = noop(str)
out = str;
end

function cvec = parse_time(str)
% str looks like this: '6/16/2009 11:04:47 AM'
assert(iscell(str));
indices = find(cellfun('isempty', str));
if ~isempty(indices)
    str{indices} = '0/0/0';
end
vec = datevec(str,0);
[numrows numcolums] = size(vec);
cvec = mat2cell(vec, ones(numrows,1), numcolums);
end

function  out = parse_duration(str)
% str looks like this: '0:00:25' or '0:00:1.2'
assert(iscell(str));
out = cell(numel(str),1);
for i=1:numel(str)
    switch length(strfind(str{i},':'))
        case 1,
            v = sscanf(str{i},'%d:%f');
            out{i} = struct('hours', 0, 'minutes', v(1), 'seconds', v(2));
            
        case 2,
            v = sscanf(str{i},'%d:%d:%f');
            out{i} = struct('hours', v(1), 'minutes', v(2), 'seconds', v(3));
                                   
        otherwise
            fprintf('Unknown notation for time interval: ''%s''\n', str{i});
            error('readFlukeFile:InvalidFields', 'Unexpected fields');
    end
end
end


function out = parse_measurement(str)
% str looks like this: '-3.926 mV DC'
assert(iscell(str));
out = cell(numel(str),1);
for i=1:numel(str)
    index = find(str{i} == ' ', 1);
    if isempty(index)
        out{i} = struct('value', nan, 'units', 'unknown');
    else
        out{i} = struct('value', str2double(str{i}(1:index)), ...
                        'units', strtrim(str{i}(index+1:end)));
    end
end
end


% ------- The CSV file is expected to look like this:
% Start Time,Elapsed Time,Interval,Total readings,Intervals,Input Events,Stop Time
% 6/16/2009 11:04:47 AM,0:00:25,0:00:01,44,26,36,6/16/2009 11:05:11 AM
% Max Time,Max,Average,Min,Min Time
% 6/16/2009 11:05:03 AM,3.595 mV DC,-0.068 mV DC,-3.926 mV DC,6/16/2009 11:05:02 AM
% Reading,Sample,Start Time,Duration,Max Time,Max,Average,Min Time,Min,Description,Stop Time
% 1,-0.230 mV DC,6/16/2009 11:04:47 AM,0:00:00.5,6/16/2009 11:04:47 AM,-0.144 mV DC,-0.235 mV DC,6/16/2009 11:04:47 AM,-0.341 mV DC,Interval,6/16/2009 11:04:47 AM

% => Variant format (note the ",,,," at the end of the header rows. Also, the duration values  
% => of the form "xx:xx.x", whereas above they are of the form "x:xx:xx.x"
% Start Time,Elapsed Time,Interval,Total readings,Intervals,Input Events,Stop Time,,,,
% 9/2/2011 12:32,0:00:59,0:00:00,70,61,18,9/2/2011 12:33,,,,
% Max Time,Max,Average,Min,Min Time,,,,,,
% 9/2/2011 12:32,1.571 mV DC,-0.040 mV DC,-1.137 mV DC,9/2/2011 12:32,,,,,,
% Reading,Sample,Start Time,Duration,Max Time,Max,Average,Min Time,Min,Description,Stop Time
% 1,1.529 mV DC,9/2/2011 12:32,00:00.4,9/2/2011 12:32,1.529 mV DC,1.529 mV DC,9/2/2011 12:32,1.529 mV DC,Interval,9/2/2011 12:32

% PARSE_FLUKE_CSV parses the above data into three structs (hdr1, hdr2
% and data). These structs have a lot of replicated information (e.g.,
% the units are replicated with every measurement) and look like the
% following:

% hdr1
% hdr1 = 
%         Start_Time: [2009 6 29 16 32 52]
%       Elapsed_Time: [1x1 struct]
%           Interval: [1x1 struct]
%     Total_readings: 63
%          Intervals: 58
%       Input_Events: 10
%          Stop_Time: [2009 6 29 16 33 50]
%          
% hdr2
% hdr2 = 
%     Max_Time: [2009 6 29 16 32 59]
%          Max: [1x1 struct]
%      Average: [1x1 struct]
%          Min: [1x1 struct]
%     Min_Time: [2009 6 29 16 32 56]
%     
% data(1)
% ans = 
%         Reading: 1
%          Sample: [1x1 struct]
%      Start_Time: [2009 6 29 16 32 52]
%        Duration: [1x1 struct]
%        Max_Time: [2009 6 29 16 32 52]
%             Max: [1x1 struct]
%         Average: [1x1 struct]
%        Min_Time: [2009 6 29 16 32 52]
%             Min: [1x1 struct]
%     Description: 'Interval'
%       Stop_Time: [2009 6 29 16 32 52]