%------------------------------------------------------------------------%
% Program : Cef2Mat (v1.06) %
%------------------------------------------------------------------------%
% Purpose 1 : To load data from a file in the ITU recommended %
% "Common Exchange Format V2.0" into MatLab workspace; %
% 2 : To save this data in a MatLab workspace file for 40x %
% faster loading next time; %
% 3 : To automatically selects the fastest file format %
% avalable when loading. %
%------------------------------------------------------------------------%
% MatLab code version: 7.3 [R2006b] %
%------------------------------------------------------------------------%
% External functions used: none %
%------------------------------------------------------------------------%
% Creation / modification history: %
% 2007-01-03 - v1.00 - B.A.Witvliet - creation as CEF 1.0 (RMDF) parser.%
% 2007-03-01 - v1.01 - B.A.Witvliet - change to CEF 2.0 format. %
% 2008-08-07 - v1.05 - B.A.Witvliet - integrated MAT-file storage/load. %
% 2008-08-20 - v1.06 - B.A.Witvliet - correct MatLab date/timestamp, %
% even if data spans several days. %
%------------------------------------------------------------------------%
% Conventions: %
% - Every new author adds to the modification history, but does never %
% erase names of earlier contributors. %
% - Arrays start with a capitol letter, single variables and single %
% variables don't. %
% - Main program names are all capitals, functions names only start %
% with a capital. %
% - Subprograms are always functions, not scripts. %
% - Global variables are avoided. %
% - This script may be used freely as long as the author is given %
% proper credit. %
%------------------------------------------------------------------------%
function [Header,Data,err,err_msg] = Cef2Mat(filename,datadir);
% filename without extension
% ------------------
% OUTPUT DATA STRUCTURE
% ------------------
% // Header data format //
% The header info is stored in a structured array named 'Header' containing
% all detected CEF 2.0 variables,e.g.:
% - Header.filetype;
% - Header.latitude;
% - Header.detector;
% etc.
% // Body data format //
% Measurement data is stored in a structured array named 'Data' containing
% time information (Data.GMT) and measured receiver voltages (Data.Level).
% Data.Level is a 3-dimensional array. Dimensions are:
% [nr of data lines] x [nr of bins] x [in case of multiscan: scan number]
% ------------------
% ERROR MESSAGES
% ------------------
% err err_msg
% 0 OK
% 1 File retrieval error (No such file)
% 2 No CEF 2.0 file
% 3 CEF file header could not be read
% 4 CEF file body could not be read
% 5 Filename should be specified without extension
% -----------------------------------------
% PREDEFINITION OF ALL OUTPUT VARIABLES
% -----------------------------------------
Header='';
Data='';
err=0;
err_msg='OK';
% -----------------------------------------
% CHECK FILENAME FORMAT
% -----------------------------------------
len=length(filename);
if filename(len-3)=='.'
err=5; err_msg='Filename should be specified without extension';
return;
end;
% -----------------------------------------
% CHECK MAT FILE PRESENCE (FOR FASTER LOADING)
% -----------------------------------------
% - Create filename with full path info -
fullname=[datadir '/' filename '.mat'];
% - If MatLab workspace file exists, load that one -
[fileID,errmsg]=fopen(fullname,'r');
if fileID~=-1
fclose(fileID);
load(fullname);
else
% -----------------------------------------
% CHECK CEF 2.0 FILE FORMAT IDENTIFICATION
% -----------------------------------------
% - Create filename with full path info -
fullname=[datadir '/' filename '.cef'];
% - Read first header line only -
[fileID,errmsg]=fopen(fullname,'r');
if fileID==-1
err=1; err_msg='No such file'; return;
end;
dataline=fgetl(fileID);
fclose(fileID);
% - Check if first line contains file format info -
if length(dataline)<12 | length(findstr(dataline,'FileType'))==0
err=2; err_msg='No CEF 2.0 file'; return;
end;
% - Check if format identification indicates CEF 2.0 -
if length(findstr(dataline,'Common Exchange Format version 2.0'))==0 & ...
length(findstr(dataline,'Common Exchange Format V2.0'))==0 & ...
length(findstr(dataline,'CEF version 2.0'))==0 & ...
length(findstr(dataline,'CEF V2.0'))==0
err=2; err_msg='No CEF 2.0 file'; return;
end;
% -----------------------------------
% READ COMPLETE FILE INTO STRING ARAY
% -----------------------------------
% - Open datafile -
fileID=fopen(fullname,'r');
% - Read all data lines until EOF -
linenr=1;
while ~feof(fileID)
dataline=fgetl(fileID);
DataS{linenr}=dataline;
linenr=linenr+1;
end;
% - Close datafile -
fclose(fileID);
% -------------
% DECODE HEADER
% -------------
% - Flag while header lines -
header=1;
% - Read header line by line -
linenr=1;
while header
dataline=DataS{linenr};
% - Check on empty line (header/body separator) -
if length(dataline)>0
% - Find field separators -
separator=strfind(dataline,' ');
% - Ignore lines w/o separator -
if length(separator)>0
% - Split line into field identifier and info field -
separator=separator(1);
identifier=dataline(1:separator-1);
info=dataline(separator+1:length(dataline));
% - Remove trailing spaces from identifier -
while strcmp(identifier(length(identifier)),' ')
identifier=identifier(1:length(identifier)-1);
end;
% - Remove preceding spaces from info field -
while length(info)>1 & strcmp(info(1),' ')
info=info(2:length(info));
end;
% - Remove trailing spaces from info field -
while length(info)>1 & strcmp(info(length(info)),' ')
info=info(1:length(info)-1);
end;
% - Identify and store all header information -
switch identifier
% - Essential info (ITU) -
case 'FileType', Header.filetype =info;
case 'LocationName', Header.locationname =info;
case 'Latitude', Header.Latitude =info;
case 'Longitude', Header.Longitude =info;
case 'FreqStart', Header.FreqStart =str2num(info)';
case 'FreqStop', Header.FreqStop =str2num(info)';
case 'AntennaType', Header.AntennaType =info;
case 'FilterBandwidth', Header.FilterBandwidth =str2num(info)';
case 'LevelUnits', Header.LevelUnits =info;
case 'Date', Header.Date =info;
case 'DataPoints', Header.DataPoints =str2num(info)';
case 'ScanTime', Header.ScanTime =str2num(info);
case 'Detector', Header.Detector =info;
% - Optional info (ITU) -
case 'Note', Header.Note =info;
case 'AntennaAzimuth', Header.AntennaAzimuth =info;
case 'AntennaElevation', Header.AntennaElevation =info;
case 'Attenuation', Header.Attenuation =info;
case 'FilterType', Header.FilterType =info;
case 'DisplayedNote', Header.DisplayedNote =info;
case 'Multiscan', Header.Multiscan =info;
% - Additional optional info (ITU) -
case 'MeasurementAccuracy', Header.MeasurementAccuracy=info;
case 'VideoFilterType', Header.VideoFilterType =info;
end;
end;
else
%- When header/body separator has been detected -
header=0;
end;
linenr=linenr+1;
end;
% - Header integrity check -
if strcmp(Header,'')
err=3; err_msg='CEF file header could not be read'; return;
end;
% - Clip and leave remainder for further processing -
DataS=DataS(linenr:length(DataS));
% ----------------
% EXTRACT TIMESTAMPS
% ----------------
% - Use start date to complete timestamps -
firstdate=Header.Date;
% - Go through all datalines -
for linenr=1:length(DataS)
% - Preparations -
dataline=DataS{linenr};
separator=strfind(dataline,',');
% - Separate timestamp from the measurement values -
gmtS=dataline(1:separator(1)-1);
Data.GMT(linenr)=datenum([firstdate ' ' gmtS],'dd-mm-yyyy HH:MM:SS');
% - Remove timestamp of dataline, leave remainder -
DataS{linenr}=dataline(separator(1)+1:length(dataline));
end;
% - Switch rows and columns -
Data.GMT=Data.GMT';
% ----------------
% CORRECT TIMESTAMP IF MULTIPLE DATES IN FILE
% ----------------
% - Detect breakpoint position between multiple dates -
dayend=find( ([0;Data.GMT]-[Data.GMT;999999]) >0)-1;
% - Add end of file position -
dayend=[dayend; length(Data.GMT)];
% - Find out how many day-ends are in this file -
nrofdays=length(dayend);
% - If multiple days then create correction array for timestamp -
if nrofdays>1
Correction=ones(dayend(1),1).*0;
for i=2:nrofdays
Correction=[Correction; ones(dayend(i)-dayend(i-1),1).*(i-1) ];
end;
% - Add appropriate date increment to timestamp -
Data.GMT=Data.GMT+Correction;
end;
% ------------
% CONVERT DATA PORTION OF FILE
% ------------
% - Convert datablock from string to array -
for linenr=1:length(DataS)
Data.Level(:,linenr,:)=str2num(DataS{linenr});;
end;
Data.Level=permute(Data.Level,[2 3 1]);
% - Integrity check body -
if strcmp(Data,'')
err=4; err_msg='CEF file body could not be read'; return;
end;
% ------------
% SAVE MAT-FILE (FOR FASTER LOADING NEXT TIME)
% ------------
% - Create filename with full path info
fullname=[datadir '/' filename '.mat'];
save(fullname,'Header','Data');
end;
