function [data] = download()
% A function for downloading xml data from www.cocomac.org. User must supply a .txt file
% containing source site names to be downloaded (see cocomac). Returns 'data' -
% a data stucture containing the following fields:
% startNode sArea sHemisphere sPDC sEC_source sPDC2 sLaminae
% endNode tArea tHemisphere tPDC tEC_source tPDC2 tLaminae fibreDensity course
% Author: James Allen
% Date: July 2006
%===============================================
% Initialise dialog box
%===============================================
dialog_fig = figure('MenuBar', 'none');
screenSize = get(0,'ScreenSize');
set(dialog_fig, 'Position',[screenSize(3)/4 screenSize(4)/4 screenSize(3)/2 screenSize(4)/2]);
set(dialog_fig, 'DefaultUicontrolUnits', 'Normalized', 'color', [0.9 0.9 0.9]);
dialog_status_label = uicontrol('Style','text', 'string','Status:', 'fontsize', 12, 'fontweight',...
'bold', 'HorizontalAlignment', 'left', 'Position', [0.05 0.9 0.15 0.05],...
'backgroundcolor', [0.9 0.9 0.9]);
dialog_status = uicontrol('Style','text', 'string','', 'fontsize', 12,...
'HorizontalAlignment', 'left', 'Position', [0.23 0.9 0.7 0.05],...
'backgroundcolor', [0.9 0.9 0.9]);
dialog_noOfChanges_label = uicontrol('Style','text', 'string',{'No. of source sites dropped:'; '(See sanitiseLog.txt)'},...
'fontsize', 10, 'HorizontalAlignment', 'center', 'Position', [0.05 0.75 0.35 0.1],...
'backgroundcolor', [0.9 0.9 0.9]);
dialog_noOfChanges = uicontrol('Style','text', 'string','', 'fontsize', 10,...
'HorizontalAlignment', 'center', 'Position', [0.41 0.78 0.13 0.05],...
'backgroundcolor', [0.9 0.9 0.9]);
dialog_panel = uipanel('Units','normalized','position', [0.05 0.2 0.9 0.5]);
dialog_brainArea_label = uicontrol('Parent', dialog_panel, 'Style','text', 'string','Brain area',...
'fontsize', 10, 'HorizontalAlignment', 'left', 'Position', [0.05 0.7 0.35 0.1],...
'backgroundcolor', [0.9255 0.9137 0.8471]);
dialog_brainArea = uicontrol('Parent', dialog_panel, 'Style','text', 'string','',...
'fontsize', 10, 'HorizontalAlignment', 'left', 'Position', [0.41 0.7 0.3 0.1],...
'backgroundcolor', [0.9255 0.9137 0.8471]);
dialog_total_brainArea = uicontrol('Parent', dialog_panel, 'Style','text', 'string','0',...
'fontsize', 10, 'HorizontalAlignment', 'center', 'Position', [0.73 0.7 0.24 0.1],...
'backgroundcolor', [0.9255 0.9137 0.8471]);
dialog_entries_label = uicontrol('Parent', dialog_panel, 'Style','text', 'string','No. of entries found:',...
'fontsize', 10, 'HorizontalAlignment', 'left', 'Position', [0.05 0.5 0.35 0.1],...
'backgroundcolor', [0.9255 0.9137 0.8471]);
dialog_entries = uicontrol('Parent', dialog_panel, 'Style','text', 'string','',...
'fontsize', 10, 'HorizontalAlignment', 'left', 'Position', [0.41 0.5 0.3 0.1],...
'backgroundcolor', [0.9255 0.9137 0.8471]);
dialog_total_entries = uicontrol('Parent', dialog_panel, 'Style','text', 'string','0',...
'fontsize', 10, 'HorizontalAlignment', 'center', 'Position', [0.73 0.5 0.24 0.1],...
'backgroundcolor', [0.9255 0.9137 0.8471]);
dialog_connections_label = uicontrol('Parent', dialog_panel, 'Style','text', 'string','No. of valid connections:',...
'fontsize', 10, 'HorizontalAlignment', 'left', 'Position', [0.05 0.3 0.35 0.1],...
'backgroundcolor', [0.9255 0.9137 0.8471]);
dialog_connections = uicontrol('Parent', dialog_panel, 'Style','text', 'string','',...
'fontsize', 10, 'HorizontalAlignment', 'left', 'Position', [0.41 0.3 0.3 0.1],...
'backgroundcolor', [0.9255 0.9137 0.8471]);
dialog_total_connections = uicontrol('Parent', dialog_panel, 'Style','text', 'string','0',...
'fontsize', 10, 'HorizontalAlignment', 'center', 'Position', [0.73 0.3 0.24 0.1],...
'backgroundcolor', [0.9255 0.9137 0.8471]);
dialog_total_label = uicontrol('Parent', dialog_panel, 'Style','text', 'string','Total:',...
'fontsize', 10, 'fontweight', 'bold', 'HorizontalAlignment', 'center', 'Position', [0.73 0.83 0.24 0.1],...
'backgroundcolor', [0.9255 0.9137 0.8471]);
dialog_finish = uicontrol('Style','text', 'string','',...
'fontsize', 10, 'HorizontalAlignment', 'left', 'Position', [0.05 0.05 0.9 0.13],...
'backgroundcolor', [0.9 0.9 0.9]);
%===============================================
%---------------------------------------------
% Timestamp start of downloading, for elapsed
% time stat at end
time_start_download = clock;
%---------------------------------------------
%%%%%%%%%%%%%%%%%%%%%%%%
%% Update dialog box %%
%% Status = reading
%%%%%%%%%%%%%%%%%%%%%%%%
set(dialog_status, 'string', 'Reading source sites file...');
drawnow
%%%%%%%%%%%%%%%%%%%%%%%%
%---------------------------------------------
% Request text file of source site names from
% user, read it into cell array sourceSites
%---------------------------------------------
[sourceSites, loadError] = fc_loadSources;
if (loadError == 1)
close(dialog_fig);
return;
end
%----------------------------------------
%----------------------------------------
% Calculate number of (UN-sanitised) source sites
%----------------------------------------
noOfUnsanitised = length(sourceSites);
if noOfUnsanitised == 0
errordlg('Error - no source sites found in sourceSites file!')
close(dialog_fig);
return;
end
%----------------------------------------
%----------------------------------------
%---- Sanitise the sourceSites cell array
% This is where sourceSites is vetted for
% illegal / invalid site names. A log
% is written to sanitiseLog.txt
%----------------------------------------
sourceSites = fc_sanitise(sourceSites); % See fc_sanitise
%----------------------------------------
%----------------------------------------
% Calculate number of (sanitised) source sites
%----------------------------------------
noOfSites = length(sourceSites);
if noOfSites == 0
errordlg('Error - no source sites found in sanitised sourceSites file!')
close(dialog_fig);
return;
end
%----------------------------------------
%%%%%%%%%%%%%%%%%%%%%%%%
%% Update dialog box %%
%% set no. sites dropped
%%%%%%%%%%%%%%%%%%%%%%%%
set(dialog_noOfChanges, 'String', num2str(noOfUnsanitised - noOfSites));
drawnow
%%%%%%%%%%%%%%%%%%%%%%%%
%----------------------------------------
% Initialisations
%----------------------------------------
log_noValidSources = 0; %For recording how many of the sourceSites we query from cocomac actually returns valid results (valid = fibre density of 1,2 or 3)
% Main data structure for storing info from cocomac
data = struct('startNode', {}, 'sArea', {}, 'sHemisphere', {}, 'sPDC', {}, 'sEC_source', {},...
'sPDC2', {}, 'sLaminae', {}, 'endNode', {}, 'tArea', {}, 'tHemisphere', {}, 'tPDC', {},...
'tEC_source', {}, 'tPDC2', {}, 'tLaminae', {}, 'fibreDensity', {}, 'course', {});
%----------------------------------------
%----------------------------------------
% Open a log file - try to open a unique
% file by adding a number to filename. Add
% a timestamp
tryOpen = false;
tryCount = 1;
while tryOpen == false
logPath = ['downloadLog' num2str(tryCount) '.txt'];
if exist(logPath) == 0
tryOpen = true;
end
tryCount = tryCount + 1;
end
try
logFile = fopen(logPath, 'at'); % a = append, t = text
catch
errordlg('Error - Couldnt open log file!')
close(dialog_fig);
return;
end
fprintf(logFile, '%s \n \n', datestr(now)); %Add a timestamp to the log
fprintf(logFile, '%s \n \n', 'Source site | No. of data entries found | No. of connections found');
%----------------------------------------
%%%%%%%%%%%%%%%%%%%%%%%%
%% Update dialog box %%
%%%%%%%%%%%%%%%%%%%%%%%%
% Set status to 'downloading'
set(dialog_status, 'String', 'Downloading XML files...');
drawnow
%%%%%%%%%%%%%%%%%%%%%%%%
% ========================================
% START OF MAIN DOWNLOAD LOOP
% ========================================
for siteCounter = 1:noOfSites
brainArea = sourceSites{siteCounter};
%-------------------------------------
%---- Download XML from www.cocomac.org
%---- see fc_urlRead
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%% ENTER COCOMAC USERNAME %%%%%%%%%%%%%
%%%%%%%%% AND PASSWORD! %%%%%%%%%%%%%
user = '';
pass = '';
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
if (isempty(user)) | (isempty(pass))
errordlg('You must enter a valid username and password!');
data = [];
return
end
url = ['http://cocomac.org/URLSearch.asp?user=' user '&password=' pass '&Search=Connectivity&DataSet=IntPrimProj&Details=&SortOrder=asc&SortBy=SourceMap&Dispmax=32767&ItemsPerPage=all&OutputType=XML_BROWSER&SearchString=(''' brainArea ''')[SOURCESITE]&tool=Matlab']; % '' is escape character for ', hence the '''
try
xmlWithHeader = urlread(url);
catch
errordlg(['Error downloading from CoCoMac - urlread failed when attempting to read ' brainArea])
close(dialog_fig);
return;
end
clear url;
%-------------------------------------
%-------------------------------------
%---- Validate XML (see fc_validate_xml)
valid = fc_validate_xml(xmlWithHeader, brainArea);
if valid == 0
close(dialog_fig);
return;
end
%-------------------------------------
%-------------------------------------
% Get no. of results from cocomac header
%-------------------------------------
noResultsStart = strfind(xmlWithHeader, '<Results>');
noResultsEnd = strfind(xmlWithHeader, '</Results>');
noResults = str2num(xmlWithHeader(noResultsStart+9 : noResultsEnd-1));
clear noResultsStart noResultsEnd;
if (noResults == []) %str2num fails, it returns an empty matrix []
errordlg(['Error working out how many data results for brain area ' brainArea ' - expected a number in XML, not found'])
close(dialog_fig);
return;
end
%%%%%%%%%%%%%%%%%%%%%%%%
%% Update dialog box %%
%%%%%%%%%%%%%%%%%%%%%%%%
% Set brain area currently downloading
% set Total brain areas +1
% set no. of entries found
% set no. of entries +1
% set connections to blank
set(dialog_brainArea, 'String', brainArea);
set(dialog_connections, 'String', '');
set(dialog_total_brainArea, 'String', num2str(str2num(get(dialog_total_brainArea, 'string')) + 1));
set(dialog_entries, 'string', num2str(noResults));
set(dialog_total_entries, 'String', num2str(str2num(get(dialog_total_entries, 'string')) + noResults));
drawnow
%%%%%%%%%%%%%%%%%%%%%%%%
%-------------------------------------
%---- Strip header
endOfHeader = strfind(xmlWithHeader,'</Header>');
if (endOfHeader == [])
errordlg(['Error finding end of header for brain area ' brainArea ' - strfind did not find </Header>'])
close(dialog_fig);
return;
end
xml = xmlWithHeader(endOfHeader + 9 : end); %The end of the header will be 9 characters after where </Header> is found
clear xmlWithHeader;
%-------------------------------------
%-------------------------------------
%---- Strip XML tags
%-------------------------------------
% XML tags removed, leaving a cell array
% containing data only
xmlData = fc_stripXmlTags(xml);
clear xml;
%-------------------------------------
%-------------------------------------
% Put xml data into data structures
%-------------------------------------
% Only continue with this procedure if there were any results returned -
% if no results, don't bother with rest of download loop. Go to next
% brainArea instead
if noResults ~= 0
[sizeXmlData, dummy] = size(xmlData);
%There should be 19 rows in xmlData for each data result - the size of xmlData should be divisible by 19
if rem(sizeXmlData, 19) ~= 0
errordlg(['Error when analysing xml from area ' brainArea ' - expected 19 xml data items per result, but rem(sizeXmlData, 19) was ' num2str(rem(sizeXmlData, 19))])
close(dialog_fig);
return;
end
%Check it also agrees with <results> field in cocomac xml
if sizeXmlData / 19 ~= noResults
errordlg(['Error when analysing xml from area ' brainArea ' - cocomac <result> field did not agree with no of results found!'])
close(dialog_fig);
return;
end
log_connectionCounter = 0;
for resultCounter = 1 : noResults
% A displacement for indexing the correct cells of xmlData when
% looping through them. 19 because each 19 cells of xmlData
% constitutues 1 set of data.
resultDisplacement = (resultCounter - 1) * 19;
%The target node may contain / \ # etc. so sanitise it. If it
%turns out to be invalid, don't store that set of data.
%Supply brainArea to fc_sanitise so it knows its dealing with
%an endNode, and will print this in the log
sanitisedTargetNode = char(fc_sanitise(cellstr(xmlData{9 + resultDisplacement}), brainArea )); %Used cellstr (converts char array to cell array) because fc_sanitise expects cell array. Used char to turn result of fc_sanitise (a cell array) back into char array
%Only proceed (store the data) if the fibre density was not 0 (meaning no
%connection) or X (meaning uncertain or unknown fibre density)
if (xmlData{17 + resultDisplacement} ~= 'X') & (xmlData{17 + resultDisplacement} ~= '0')
%Only proceed (store the data) if the end (target) node survives the
%sanitisation procedure. If the endNode is not returned
%from fc_sanitise, the endnode is not valid.
if (~isempty(sanitisedTargetNode))
%--------------------------------------
%Extract the start node (e.g. V1) from study ID + start
%node (e.g. AM02-V1) in xmlData. Replace the xmlData field
%with just the start node.
dashPos = strfind(xmlData{1 + resultDisplacement}, '-'); % 1 because it is the first field in
if (isempty(dashPos))
error('Ooops - couldnt find a dash in study ID + startnode field');
end
xmlData{1 + resultDisplacement} = xmlData{1 + resultDisplacement}(dashPos+1:end);
clear dashPos;
%--------------------------------------
%--------------------------------------
%Do the same with the end node, but sanitise the endnode
%aswell (start node is already sanitised)
dashPos = strfind(xmlData{9 + resultDisplacement}, '-');
if (isempty(dashPos))
error('Ooops - couldnt find a dash in study ID + endNode field');
end
xmlData{9 + resultDisplacement} = xmlData{9 + resultDisplacement}(dashPos+1:end);
clear dashPos;
%--------------------------------------
data(end+1, 1).startNode = xmlData{1 + resultDisplacement}; % 1 = start (source) node
data(end, 1).sArea = xmlData{2 + resultDisplacement}; % 2 = source area
data(end, 1).sHemisphere = xmlData{3 + resultDisplacement}; % 3 = source hemisphere
data(end, 1).sPDC = xmlData{4 + resultDisplacement}; % 4 = source PDC
data(end, 1).sEC_source = xmlData{5 + resultDisplacement}; % 5 = source EC Source
data(end, 1).sPDC2 = xmlData{6 + resultDisplacement}; % 6 = source PDC2
data(end, 1).sLaminae = xmlData{7 + resultDisplacement}; % 7 = source laminae
%xmlData{8} is just a dash
data(end, 1).endNode = xmlData{9 + resultDisplacement}; % 9 = end (target) node
data(end, 1).tArea = xmlData{10 + resultDisplacement}; % 10 = target area
data(end, 1).tHemisphere = xmlData{11 + resultDisplacement}; % 11 = target hemisphere
data(end, 1).tPDC = xmlData{12 + resultDisplacement}; % 12 = target PDC
data(end, 1).tEC_source = xmlData{13 + resultDisplacement}; % 13 = target EC source
data(end, 1).tPDC2 = xmlData{14 + resultDisplacement}; % 14 = target PDC2
data(end, 1).tLaminae = xmlData{15 + resultDisplacement}; % 15 = target laminae
%xmlData{16} is just a dash
data(end, 1).fibreDensity = xmlData{17 + resultDisplacement}; % 17 = fibre density
%xmlData{18} is just a dash
data(end, 1).course = xmlData{19 + resultDisplacement}; % 19 = fibre course
% Increment a connectionCounter (for saving as a log)
log_connectionCounter = log_connectionCounter + 1;
end %End of condition: endNode must be valid (survives fc_sanitise)
end %End of condition: fibre density is not X or 0
end %end of for loop going through xmlData and putting it into data structure
%%%%%%%%%%%%%%%%%%%%%%%%
%% Update dialog box %%
%%%%%%%%%%%%%%%%%%%%%%%%
% set no. of connections
set(dialog_connections, 'string', num2str(log_connectionCounter));
set(dialog_total_connections, 'String', num2str(str2num(get(dialog_total_connections, 'string')) + log_connectionCounter));
drawnow
%%%%%%%%%%%%%%%%%%%%%%%%
%disp(['Number of valid connections: ' num2str(log_connectionCounter)])
%-----------------------------------------
%------ Increment log_noValidSources
if log_connectionCounter > 0
log_noValidSources = log_noValidSources + 1;
end
%-----------------------------------------
%-----------------------------------------
%------ SAVE LOG (append) for THIS brain area
% downloadLog.txt
bytesWritten = fprintf(logFile, '%s', [brainArea ', ' num2str(noResults) ', ' num2str(log_connectionCounter)]);
if (bytesWritten == 0)
msgbox(['Warning - 0 bytes written to log file when writing ' brainArea '!'], 'Warning', 'warn')
close(dialog_fig);
return;
end
clear bytesWritten;
fprintf(logFile, '\n'); %New line
%------------------------------------------
end % End of condition: if there were any results returned
end % End of the main downloading loop - going through all entries in sourceSites (sanitised), downloading xml, storing data.
%======================================
%==== END OF MAIN DOWNLOADING LOOP
%======================================
%--------------------------------------
%-------Save the final log entry
fprintf(logFile, '\n'); %New line
fprintf(logFile, '\n'); %New line
fprintf(logFile, '%s \n', ['Downloaded ' num2str(noOfSites) ' xml files and saved ' get(dialog_total_connections, 'string') ' connections.']);
fclose(logFile);
clear logFile;
%----------------------------------------
% Timestamp end of download
time_end_download = clock;
%%%%%%%%%%%%%%%%%%%%%%%%
%% Update dialog box %%
%%%%%%%%%%%%%%%%%%%%%%%%
% finish. Please select save
set(dialog_status, 'string', 'Finishing...');
set(dialog_finish, 'string', ['Finished! Downloaded ' num2str(noOfSites) ' xml files and saved ' get(dialog_total_connections, 'string') ' connections. Elapsed time: ' num2str(etime(time_end_download, time_start_download)) ' seconds. See download log: ' logPath]);
drawnow
%%%%%%%%%%%%%%%%%%%%%%%%
%-----------------------------------
% Make an entry in the GUI_log
fc_saveLog(['Downloaded ' num2str(noOfSites) ' xml files and saved ' get(dialog_total_connections, 'string') ' connections. Elapsed time: ' num2str(etime(time_end_download, time_start_download)) ' seconds. See download log: ' logPath]);
%--------------------------------------
%------Save the data structure
% Ask the user where to save mat file
[save_name, save_path] = uiputfile('.mat', 'Save network data');
saveFile = [save_path save_name];
% If the user did not select a file..
if isequal(save_path,0) | isequal(save_name,0)
msgbox('Did not save network data!', 'Warning,', 'warn')
% Otherwise...
else
try
save (saveFile, 'data');
catch
errordlg(['Error saving data structure to ' saveFile ': ' lasterr])
end
end
%%%%%%%%%%%%%%%%%%%%%%%%
%% Update dialog box %%
%%%%%%%%%%%%%%%%%%%%%%%%
% Set status to 'downloading'
set(dialog_status, 'String', 'Finished!');
drawnow
%%%%%%%%%%%%%%%%%%%%%%%%
%--------------------------------------
% return data
return;
