function metaStruct = TDMS_preprocessFile(fid,tdmsFileName,params)
%TDMS_preprocessFile Populates props and gets segment information
%
% NOTE: This file shouldn't be called directly by the user
%
% metaStruct = TDMS_preprocessFile(fid,tdmsFileName,params)
%
% INPUTS
% =======================================================================
% fid : file id of opened tdms file, may be null if INDEX_DEBUG is true
% tdmsFileName : full path to tdms file OR tdms index file for INDEX_DEBUG
% params : all optional inputs from TDMS_readTDMSFile
%
% OUTPUTS
% =======================================================================
% metaStruct :
% structure with fields:
%
% The following 3 variables are all the same length, and have one index
% for each unique object. Each object may be one of the following types,
% file root, group object, or channel object. Only channel objects may
% have data.
%
% rawDataInfo : (structure array)
% NOTE: This holds the most recent info for an object, and can be
% changed on a per segment basis, segInfo is used to hold
% information that is needed per segement
%
% .lengthOfIndex - length of raw data index (not used)
% .dataType - enumerated dataType (Labview)
% .dimensionData - currently ignored (should be 1d)
% .numberOfValues - # of values to read each time
% .totalSizeBytes - # of bytes per values
% .numberProperties - # of properties defined for the object
% .infoSet - a logical on whether or not this data has
% been set (for error checking)
% .propNames - a cell array for definining property names
% .propValues - cell array for holding property values
%
% numberDataPoints : an array which specifies the # of data points for
% each object (used for preallocation)
% objectNameList : (cell array of strings) names of all objects in the
% TDMS file
%
% Info for each segment ...
%
% segInfo : (struct array)
% .rawPos - position of raw data for that segment
% .kTocNewObjList - flag on whether a new channel list was
% created or not
% .objOrder - array specifying indices of the objects to assign
% data to, order specifies read order, and values
% are indexed
% into the objectNameList & rawDataInfo struct
% .nRawObjects - length of objOrder
% .nSamplesRead - array, # of samples to read for each object
% .isInterleaved - logical, whether that segment uses interleaved
% data or not
% .isBigEndian - logical
% .nChunks - # of times to read data using the parameters
% above before moving onto the next segment
%
% See also: TDMS_processLeadIn, TDMS_getDataSize, TDMS_getPropValue
%PARAMETERS - passed in from TDMS_readTDMSFile
%==========================================================================
UTC_DIFF = params.UTC_DIFF;
MAX_NUM_OBJECTS = params.MAX_NUM_OBJECTS;
MAX_NUM_PROPS = params.MAX_NUM_PROPS;
N_SEGS_GUESS = params.N_SEGS_GUESS;
N_SEGS_INC = params.N_SEGS_INC;
DEBUG = params.DEBUG;
DATE_STR_FORMAT = params.DATE_STR_FORMAT;
UNICODE_FORMAT = params.STRING_ENCODING;
USE_INDEX = params.USE_INDEX;
INIT_CHUNK_SIZE = params.INIT_CHUNK_SIZE;
TDMS_INDEX_EXT = params.TDMS_INDEX_EXT;
MACHINE_FORMAT = params.MACHINE_FORMAT;
STRING_ENCODING = params.STRING_ENCODING;
CURRENT_VERSION = params.CURRENT_VERSION;
INDEX_DEBUG = params.INDEX_DEBUG;
%DONT CHANGE THESE
%==========================================================================
LEAD_IN_LENGTH = 28; %# of bytes of lead in
%FIGURING OUT WHICH FILE TO READ
%==========================================================================
[tdmsPathToFile,tdmsNameOnly] = fileparts(tdmsFileName);
if isempty(params.META_STRUCT)
isIndexFID = false;
if USE_INDEX || INDEX_DEBUG
if INDEX_DEBUG
indexFile = tdmsFileName;
else
indexFile = fullfile(tdmsPathToFile,[tdmsNameOnly TDMS_INDEX_EXT]);
end
if exist(indexFile,'file')
fid = fopen(indexFile,'r',MACHINE_FORMAT,STRING_ENCODING);
isIndexFID = true;
else
%Just use the tdms file, which we have already tested to exist
if INDEX_DEBUG
%NOTE: With INDEX_DEBUG we explicitly passed in the index
%file to read and parse (generally for debugging purposes
%so if it doesn't exist, then we have a problem
error('Specified tdms_index file doesn not exist')
end
end
end
else
%Quits early if meta struct input is valid
%======================================================================
metaStruct = params.META_STRUCT;
if ~isstruct(metaStruct) || ~isfield(metaStruct,'version') || ~isfield(metaStruct,'fileName')
error('The META_STRUCT parameter passed should be a structure with fields "version" and "fileName"')
end
if metaStruct.version ~= CURRENT_VERSION
error('Mismatch in version creation, version of struct %d, please run with latest version: %d')
end
if ~strcmpi(metaStruct.fileName,tdmsNameOnly)
fprintf('Filename from metaStruct: %s\n',metaStruct.fileName);
fprintf('Filename from input: %s\n',tdmsNameOnly);
error('Mismatch in filenames, see printout above')
end
return
end
if isIndexFID
lastLetter = double('h'); %used for .tdms_index files
else
lastLetter = double('m'); %used for .tdms files
end
%INITIALIZE OUTPUTS
%==========================================================================
curNumTotalObjects = 0; %Current # of objects
numberDataPoints = zeros(1,MAX_NUM_OBJECTS);
objectNameList = cell(1,MAX_NUM_OBJECTS); %names of all objects
rawDataInfo = initRawInfoStruct(MAX_NUM_PROPS,MAX_NUM_OBJECTS);
segInfo = initSegStruct(N_SEGS_GUESS);
%==========================================================================
%TEMP VARIABLES
%==========================================================================
objectHasRawData = false(1,MAX_NUM_OBJECTS); %This is needed for adding
%# of data points together
ranOnce = false;
nSegs = 0;
%Get eof & return to start
fseek(fid,0,1);
eofPosition = ftell(fid);
fseek(fid,0,-1);
%START OF READING
%==========================================================================
curPosCounter = 0; %Used to keep track of where in the actual tdms file we
%would be at various points, this is used for parsing verification as well
%as assisting with more complicated reads
while ftell(fid) ~= eofPosition
nSegs = nSegs + 1;
if nSegs > length(segInfo)
segInfo = [segInfo initSegStruct(N_SEGS_INC)]; %#ok<AGROW>
end
if DEBUG
disp('------ START OF DEBUG -------')
fprintf(2,'CURRENT SEGMENT: %d\n',nSegs);
fprintf(2,'Current file position: %d\n',curPosCounter);
end
%LEAD IN HANDLING
%======================================================================
[flags,info,eof_error] = TDMS_processLeadIn(fid,lastLetter);
if eof_error
%This should only happen once at the end
fprintf(2,['WARNING: File was not closed properly.\n' ...
'Data will most likely be missing at the end of the file\n']);
nSegs = nSegs - 1;
break
end
curPosCounter = curPosCounter + LEAD_IN_LENGTH + info.segLength;
segInfo(nSegs).rawPos = curPosCounter - info.segLength + info.metaLength;
if ~ranOnce
%NOTE: This might be false if no channels are defined ...
if flags.kTocNewObjList == false
error('the kTocNewObjList was false when first run, this is not expected')
end
ranOnce = true;
end
%======================================================================
%======================================================================
% META INFORMATION PROCESSING
%======================================================================
if flags.hasMetaData
%Get # of changed objects
numNewObjInSeg = fread(fid,1,'uint32');
%Reinitialize order list if new
if flags.kTocNewObjList
curObjListCount = 0;
objOrder = zeros(1,2*numNewObjInSeg);
nValuesRead = zeros(1,2*numNewObjInSeg);
%NOTE: I padded this by doubling the #, we might append extra
%channels in subsequent reads, I currently don't resize this
%...
end
for iNewObject = 1:numNewObjInSeg
%1) GET OBJECT PATH
%-------------------------------
objPathLength = fread(fid,1,'uint32');
temp = fread(fid,objPathLength,'*uint8');
objName = native2unicode(temp,UNICODE_FORMAT)'; %#ok<*N2UNI>
%POPULATE ALL OBJECT LIST
%----------------------------------------
objIndex = find(strcmp(objectNameList(1:curNumTotalObjects),objName),1);
if isempty(objIndex)
curNumTotalObjects = curNumTotalObjects + 1;
if curNumTotalObjects > length(rawDataInfo)
rawDataInfo = [rawDataInfo ...
initRawInfoStruct(MAX_NUM_PROPS,MAX_NUM_OBJECTS)]; %#ok<AGROW>
numberDataPoints = [numberDataPoints zeros(1,MAX_NUM_OBJECTS)]; %#ok<AGROW>
objectNameList = [objectNameList cell(1,MAX_NUM_OBJECTS)]; %#ok<AGROW>
objectHasRawData = [objectHasRawData false(1,MAX_NUM_OBJECTS)]; %#ok<AGROW>
end
objIndex = curNumTotalObjects;
objectNameList{curNumTotalObjects} = objName;
end
%RAW DATA INDEX PROCESSING
%---------------------------------------------
curPos = ftell(fid);
rawDataIndexLength = fread(fid,1,'uint32');
rawDataInfo(objIndex).lengthOfIndex = rawDataIndexLength;
if DEBUG
fprintf(2,'RawDataLength: %d\n',rawDataIndexLength);
fprintf(2,'CurrentPos: %d\n',curPos);
end
switch rawDataIndexLength
case 0 %Same as previous
if rawDataInfo(objIndex).infoSet == false
error('Channel %s set to use previous rawDataIndex but this channel is new',objName)
end
%NOTE: "same as previous segment" apparently means
%"same as the previous one with data ..."
if rawDataInfo(objIndex).numberOfValues > 0 && ~objectHasRawData(objIndex)
objectHasRawData(objIndex) = true;
end
case 2^32-1 %no raw data
rawDataInfo(objIndex).infoSet = true;
objectHasRawData(objIndex) = false;
otherwise
objectHasRawData(objIndex) = true;
rawDataInfo(objIndex).infoSet = true;
%DATA TYPE HANDLING
%------------------------------------------------------
dataType = fread(fid,1,'uint32');
if rawDataInfo(objIndex).infoSet && dataType ~= rawDataInfo(objIndex).dataType && numberDataPoints(objIndex) > 0
error('Raw data type for channel %s has changed from %d to %d',...
objName,rawDataInfo(objIndex).dataType,dataType)
else
rawDataInfo(objIndex).dataType = dataType;
end
%DATA SIZE HANDLING
%-----------------------------------------------------
rawDataInfo(objIndex).dimensionData = fread(fid,1,'uint32');
if rawDataInfo(objIndex).dimensionData ~= 1
error('Code doesn''t yet handle non 1D data')
end
rawDataInfo(objIndex).numberOfValues = fread(fid,1,'uint64');
%RawDaqMX
%------------------------------------------------------
%NOTES:
if dataType == 2^32-1
%JIM CODE IN PROGRESS
rawDataInfo(objIndex).isRawDAQmx = true; %We can
% % % %post process this to convert from bits to an
% % % %actual value
% % %
% % % %FORMAT:
% % % %1) -
% % % daqMXVersion = rawDataIndexLength;
% % % if ~ismember(daqMXVersion,[4713 4714])
% % % error('Unexpected version: %d',daqMXVersion)
% % % end
% % %
% % %
% % % %2) Let's get the remaining # of bytes
% % % %This seems to be 32 ...
% % %
% % % % % wtf = fread(fid,8,'uint32');
% % % % % disp(wtf)
% % % %4713
% % % %13000012
% % % %15000018
% % % %? -> 8th indicates data type? OR -> bytes per sample
% % %
% % % %4714
% % % %1 0 0 0 256 256 512
% % %
% % % %3) Update datetype - where the heck is it?
% % % %rawDataInfo(objIndex).dataType = dataType;
else %Non RawDAQmx type
if rawDataInfo(objIndex).dataType == 32
%If string, size is specified by an additional field
rawDataInfo(objIndex).totalSizeBytes = fread(fid,1,'uint64');
else
rawDataInfo(objIndex).totalSizeBytes = ...
rawDataInfo(objIndex).numberOfValues*TDMS_getDataSize(dataType);
end
%Another chance to check correct reading
if curPos + rawDataIndexLength ~= ftell(fid)
error(['Raw Data Index length was incorrect: %d stated vs %d observed,' ...
'likely indicates bad code or a bad tdms file'],rawDataIndexLength,ftell(fid) - curPos)
end
end
if DEBUG
fprintf(2,'nSegs: %d\n',nSegs);
fprintf(2,'objName: %s\n',objName);
end
end
%--------------------------------------------------------------
%POPULATE ORDER TO RETRIEVE RAW DATA
%--------------------------------------------------------------
if objectHasRawData(objIndex)
appendToList = false;
if flags.kTocNewObjList
appendToList = true;
else %Only append if not currently specified
I_objOrder = find(objOrder(1:curObjListCount) == objIndex,1);
if isempty(I_objOrder)
appendToList = true;
else
nValuesRead(I_objOrder) = rawDataInfo(objIndex).numberOfValues;
end
end
%NOTE: No overflow code in place yet, however we do
%initialize with twice the # of objects specified to be
%in a segement when a new list is created, new segments
%might add more objects to the list
if appendToList
curObjListCount = curObjListCount + 1;
objOrder(curObjListCount) = objIndex;
nValuesRead(curObjListCount) = rawDataInfo(objIndex).numberOfValues;
end
end
%PROPERTY HANDLING
%--------------------------------------------------------------
numberProperties = fread(fid,1,'uint32');
%Below is the # of props already assigned to that channel
nPropsChan = rawDataInfo(objIndex).numberProperties;
curProps = rawDataInfo(objIndex).propNames;
for iProp = 1:numberProperties
propNameLength = fread(fid,1,'uint32');
temp = fread(fid,propNameLength,'*uint8');
propName = native2unicode(temp,UNICODE_FORMAT)';
propDataType = fread(fid,1,'uint32');
propIndex = find(strcmp(curProps(1:nPropsChan),propName),1);
if isempty(propIndex)
%Updates needed for new properties
nPropsChan = nPropsChan + 1;
propIndex = nPropsChan;
curProps{propIndex} = propName;
rawDataInfo(objIndex).propNames{propIndex} = propName;
end
%Update value
propValue = TDMS_getPropValue(fid,propDataType,UTC_DIFF,DATE_STR_FORMAT);
rawDataInfo(objIndex).propValues{propIndex} = propValue;
end
rawDataInfo(objIndex).numberProperties = nPropsChan;
if DEBUG
fprintf(2,'end of index position: %d\n',ftell(fid));
end
end
end
%======================================================================
% END OF META DATA PROCESSING
%RAW DATA SAMPLE COUNTING
%======================================================================
segInfo(nSegs).kTocNewObjList = flags.kTocNewObjList;
byteSizeRaw = info.segLength - info.metaLength;
if ~flags.hasRawData || byteSizeRaw == 0
segInfo(nSegs).nChunks = 0;
else
segInfo(nSegs).objOrder = objOrder(1:curObjListCount);
segInfo(nSegs).nRawObjects = curObjListCount;
segInfo(nSegs).nSamplesRead = nValuesRead(1:curObjListCount);
segInfo(nSegs).isInterleaved = flags.isInterleaved;
segInfo(nSegs).isBigEndian = flags.isBigEndian;
%# OF CHUNK PROCESSING
%------------------------------------------------------------------
totalBytesPerChunk = sum([rawDataInfo(objOrder(1:curObjListCount)).totalSizeBytes]);
nChunks = byteSizeRaw/totalBytesPerChunk;
%Some error checking
%------------------------------------------
if DEBUG
fprintf(2,'nChunks: %d\n',nChunks);
fprintf(2,'nSamplesRead: %s\n',mat2str(nValuesRead(1:curObjListCount)));
fprintf(2,'totalBytesPerChunk: %d\n',totalBytesPerChunk);
fprintf(2,'byteSizeRaw: %d\n',byteSizeRaw);
end
if nChunks ~= floor(nChunks)
error(['The remaining data doesn''t split evently into' ...
' chunks, estimated # of chunks: %d'],nChunks)
end
chunkByteOffset = 0;
%Increment the number of data points
for iObject = 1:curObjListCount
curIndex = objOrder(iObject);
nSamplesReadCurObject = rawDataInfo(curIndex).numberOfValues;
%Apparently some writers don't stripe the objects when some
%objects don't have any data and some do ...
if nSamplesReadCurObject > 0
%This allows us to grow these values if we haven't sufficiently preallocated
if rawDataInfo(curIndex).chunkIndex + nChunks > rawDataInfo(curIndex).chunkLength
rawDataInfo(curIndex).dataMatrix = [rawDataInfo(curIndex).dataMatrix; zeros(INIT_CHUNK_SIZE,3)];
rawDataInfo(curIndex).chunkLength = rawDataInfo(curIndex).chunkLength + INIT_CHUNK_SIZE;
end
%DataMatrix:
%==============================================================
%This information is used for reading parts of an object during
%a single read instead of the entire object
%Column 1, file position
indices = (rawDataInfo(curIndex).chunkIndex+1):(rawDataInfo(curIndex).chunkIndex+nChunks);
rawDataInfo(curIndex).dataMatrix(indices,1) = ...
segInfo(nSegs).rawPos + chunkByteOffset + (0:totalBytesPerChunk:(nChunks-1)*totalBytesPerChunk);
%Column 2, first sample at that position
rawDataInfo(curIndex).dataMatrix(indices,2) = ...
numberDataPoints(curIndex) + (0:nSamplesReadCurObject:(nChunks-1)*nSamplesReadCurObject) + 1;
%Column 3, segment number
rawDataInfo(curIndex).dataMatrix(indices,3) = nSegs;
chunkByteOffset = chunkByteOffset + rawDataInfo(curIndex).totalSizeBytes;
numberDataPoints(curIndex) = numberDataPoints(curIndex) + nSamplesReadCurObject*nChunks;
rawDataInfo(curIndex).chunkIndex = rawDataInfo(curIndex).chunkIndex+nChunks;
end
end
%nChunksAll = nChunksAll + nChunks;
segInfo(nSegs).nChunks = nChunks;
%This needs to be handled in some manner for RawDaqMx
if ~isIndexFID
fseek(fid,byteSizeRaw,'cof');
end
end
%Addition for raw data (needs to be fixed)
%In general, the tdms_index requires skipping some meta data
%I think this is only needed for raw data (I need to flush this out
%better as it will cause an error when reading index files for regular
%tdms files)
% if isIndexFID
% fseek(fid,segInfo(nSegs).rawPos,'bof');
% end
end
%Trim output:
%==========================================================================
rawDataInfo = rawDataInfo(1:curNumTotalObjects);
for iObject = 1:length(rawDataInfo)
nProps = rawDataInfo(iObject).numberProperties;
rawDataInfo(iObject).propNames = rawDataInfo(iObject).propNames(1:nProps);
rawDataInfo(iObject).propValues = rawDataInfo(iObject).propValues(1:nProps);
%Add extra value for later processing
if rawDataInfo(iObject).chunkIndex ~= 0
rawDataInfo(iObject).chunkIndex = rawDataInfo(iObject).chunkIndex + 1;
rawDataInfo(iObject).dataMatrix(rawDataInfo(iObject).chunkIndex,2) = numberDataPoints(iObject) + 1;
end
end
numberDataPoints = numberDataPoints(1:curNumTotalObjects);
objectNameList = objectNameList(1:curNumTotalObjects);
segInfo = segInfo(1:nSegs);
metaStruct = struct(...
'eof_error', eof_error,...
'numberDataPoints', numberDataPoints,...
'objectNameList', {objectNameList},...
'segInfo', segInfo,...
'rawDataInfo', rawDataInfo,...
'fileName', tdmsNameOnly,...
'version', CURRENT_VERSION);
if isIndexFID
%If using the index file to parse meta, close it
fclose(fid);
end
end
function segStruct = initSegStruct(nSegs)
%initSegStruct
%
% segStruct = initSegStruct(nSegs)
%
segStruct = struct(...
'rawPos',repmat({0},[1 nSegs]),...
'kTocNewObjList',0,...
'objOrder',[],...
'nRawObjects',0,...
'nSamplesRead',[],...
'isInterleaved',false,...
'isBigEndian',false,...
'nChunks',false);
end
function rawInfoStruct = initRawInfoStruct(MAX_NUM_PROPS,MAX_NUM_OBJECTS)
rawInfoStruct = struct( ...
'isRawDAQmx', false, ...
'lengthOfIndex', 0,...
'dataType', 0,...
'dimensionData', 0,...
'numberOfValues', 0,...
'totalSizeBytes', 0,... %Only valid for strings
'numberProperties', 0,...
'chunkIndex', 0,...
... %Could initialize to zero, this would save space
... %with non-raw data objects
'chunkLength', 0,...
... %SEE ALSO: chunk loop above for resizing
'dataMatrix', zeros(0,3),...
...%Position #
...%1st Sample #
...%Seg #
'infoSet', false, ...
'propNames', repmat({cell(1,MAX_NUM_PROPS)},1,MAX_NUM_OBJECTS),...
'propValues', repmat({cell(1,MAX_NUM_PROPS)},1,MAX_NUM_OBJECTS));
end
