% This script will scrape vote, view and remix counts from 
% all users with more than 3 submissions.  You will be asked
% whether you want to save the data to a mat file (yes|no). 
% "Yes" will prompt a dialog to choose a folder.  Whether 
% you save the data or not, the script will continue to 
% produce summary plots of vote and view counts.  
%
% Code blocks
%   Block 1: scrapes data and offers to save the table T.
%   Other blocks: Can be run independently to produce 
%       plots of data. See descriptions at the top of
%       each block.
%   Final block: Pulls and plots data across multiple
%       saved tables. 
%
% Written on Oct 13, 2021; Adam Danz using Matlab R2021b,
% also tested in R2019b.  
%
% Version 2.0 Oct 31, 2021

%% SCRAPE LEADER BOARD FOR AUTHOR IDs
clear; clc; close all

% Base URL
URL = 'https://www.mathworks.com/matlabcentral/communitycontests/contests/4'; 

% Loop through all pages of leaderboard
T = table(); 
page = 0; 
results = @(str)regexp(str,'Results \d+ - (\d+) of (\d+)','once','tokens'); % regexp to search for "results xx - xx of xxx".
isLastPage = @(resultRange)numel(unique(str2double(resultRange)))==1; % returns 1 if last page
while ~isinf(page)
    page = page+1;
    fprintf('Getting user IDs from leaderboard page %d\n',page)
    URLPG = [URL,'/rankings',sprintf('?page=%d',page)];
    D = webread(URLPG);

    % Get all user IDs on current page of leaderboard
    % This will skip users with less than 4 submissions.  
    userIDs = regexp(D,'contributor_id=(\d+)"','tokens');
    userIDs = [userIDs{:}];

    % Loop through each user ID "view all" to scrape data from each submission
    nUsers = numel(userIDs); 
    for j = 1:nUsers
        fprintf('Getting data from user %d/%d (id=%s)\n',j,nUsers,userIDs{j})
        tempURL = [URL, '/entries?contributor_id=', userIDs{j}];
        txt = webread(tempURL); % web(tempURL) to open page
        % Get User Name
        username = strtrim(regexp(txt,'(?<=Created by )(.*?)(?= (about |on |less)?\d+)','once','tokens'));

        % Loop through pages of submissions for user j
        userPage = 0;
        while ~isinf(userPage)
            % Update URL with page number
            userPage = userPage+1;
            userPageURL = [tempURL,sprintf('&page=%d',userPage)];
            ptxt = webread(userPageURL); % web(userPageURL) to open page
            % Get range of results on this page
            resultRange = results(ptxt); % ex.: Results 1 - 32 of 59
            % Get votes|views|remixes
            dataStr = regexp(ptxt,'<span>(\d+)<\/span>','tokens');
            data = str2double([dataStr{:}]);
            assert(mod(numel(data),3)==0, '"data" is not a multiple of 3') % there should be 3 values per submission
            tempT = [table(repmat(categorical(username),numel(data)/3,1),...
                'VariableNames',"User"), ...
                array2table(reshape(data',3,[])',...
                'VariableNames',["Votes","Views","Remixes"])];
            T = [T;tempT];   %#ok<AGROW>
            % Check for more pages of submissions for user j
            if isLastPage(resultRange)
                userPage = inf;
            end
        end
    end
    % Get number of results shown on current leader page
    leaderResults = results(D); % ex.: Results 1 - 32 of 59
    if isLastPage(leaderResults)
        page = inf;
    end
end
fprintf('Done. Data in table "T".\n')

% Offer to save table to file saved with timestamp
yn = questdlg('Save table to file?','Save');
if strcmpi(yn,'Yes')
    selectedDir = uigetdir(); 
    if ischar(selectedDir)
        filename = sprintf('LeaderboardData_%s',datestr(now,'yymmdd_HHMM'));
        fullfilename = fullfile(selectedDir,filename); 
        save(fullfilename,'T')
        fprintf('Data saved <a href="matlab: winopen(''%s'') "> [open dir.] </a>\n', selectedDir)  % windows 
    end
end

%% Summarize data per user
% This section can be run indpendently when table T is available. 
% It produces a stacked plot summarizing sum and mean votes, views, and remixes.

TSummary = groupsummary(T,'User',{'sum','mean'});
fig1 = figure(); 
% tlo = tiledlayout(4,1,'TileSpacing','compact','Padding','compact');
% ax1 = nexttile(tlo);
sh=stackedplot(fig1,TSummary,TSummary.Properties.VariableNames(2:end),'-o');
sh.DisplayLabels{1} = '# submissions';
grid on
stackedAxes = findobj(sh.NodeChildren, 'Type','Axes');
set(stackedAxes, 'XTick', 1:height(TSummary))
set(stackedAxes(1), 'XTickLabels', string(TSummary.User))
title(sh,'Leaderboard summary')

%% Show number of submissions per user
% This section can be run independently when table T is available.  
% It produces a figure showing the distribution of number of submissions. 

minNum = 5; % minimum number of submissions to conciser
TSummaryCount = groupsummary(T,'User',{'sum','mean'});
countIdx = TSummaryCount.GroupCount<minNum;
rmUsers = string(TSummaryCount.User(countIdx));
TSummaryCount(countIdx,:) = [];
TSummaryCount.User = removecats(TSummaryCount.User, rmUsers);
fig4 = figure(); 
ax=axes(fig4);
[~, sortIdx] = sort(TSummaryCount.GroupCount);
sortedCats = reordercats(TSummaryCount.User,string(TSummaryCount.User(sortIdx)));
bar(ax,sortedCats, TSummaryCount.GroupCount)
grid(ax,'on')
ylabel(ax,'Count')
title(ax,'Number of submissions')
subtitle(ax,sprintf('Users with >=%d submissions',minNum))

%% Rank participants by number of views
% This section can be run independently when table T is available.  
% It produces a figure showing the distribution of number of views by participant. 

minNum = 150; % minimum number of views to conciser
TSummaryViews = groupsummary(T,'User',{'sum','mean'});
viewIdx = TSummaryViews.sum_Views<minNum;
rmUsers = string(TSummaryViews.User(viewIdx));
TSummaryViews(viewIdx,:) = [];
TSummaryViews.User = removecats(TSummaryViews.User, rmUsers);
fig6 = figure(); 
ax=axes(fig6);
[~, sortIdx] = sort(TSummaryViews.sum_Views);
sortedCats = reordercats(TSummaryViews.User,string(TSummaryViews.User(sortIdx)));
bar(ax,sortedCats, TSummaryViews.sum_Views)
hold(ax,'on')
h=plot(ax,sortedCats, TSummaryViews.mean_Views, 'w_', 'LineWidth', 2);
grid(ax,'on')
ylabel(ax,'Count')
title(ax,'Number of views')
legend(h,'Mean','Location','NorthWest','Color',[.7 .7 .7])
subtitle(ax,sprintf('Users with >=%d views',minNum))

%% Vote trendlines
% This section can be run indpendently when table T is available. 
% It produces a figure showing voting trend lines for all users with 
% more than a threshold number of submissions (minSubs).  

% Include users with more than a threshold number of submissions
TSummary = groupsummary(T,'User',{'sum','mean'});
minSubs = 8; 
isIncluded = TSummary.GroupCount >= minSubs;   
Tsub = TSummary(isIncluded,:);
% One line per user showing vote profile across submissions. 
usersUnq = unique(Tsub.User);
nUsers = height(Tsub);
fig2 = figure();
tlo = tiledlayout(fig2,'flow','TileSpacing','Compact','Padding','Compact');
for i = 1:nUsers 
    votes = sort(T.Votes(T.User==usersUnq(i)));
    ax = nexttile(tlo);
    plot(ax,votes,'-o')
    title(ax,usersUnq(i),'Interpreter','none')
    xlim(ax,[0,numel(votes)])
    ylim(ax,[0,inf])
    text(ax,max(xlim(ax)),min(ylim(ax)),...
        sprintf('Range [%d,%d] ',votes([1,end])),...
        'HorizontalAlignment','Right',...
        'VerticalAlignment','Bottom','Fontsize', 8)
    grid(ax,'on')
end
xlabel(tlo,'Submissions (sorted by vote count)','FontSize',16)
ylabel(tlo,'Votes for each submission','FontSize',16)
title(tlo,'Vote profiles across submissions','FontSize',18)
subtitle(tlo,sprintf('Participants with more than %d submissions',minSubs-1))

%% Correlation between views and votes
% This section can be run indpendently when table T is available. 
% It produces a figure showing the correlation between views and votes 
% for submissions from users with more than a threshold amount of 
% submissions (minSubs).  

% Include users with more than a threshold number of submissions
TSummary = groupsummary(T,'User',{'sum','mean'});
minSubs = 8; 
isIncluded = TSummary.GroupCount >= minSubs;   
Tsub = TSummary(isIncluded,:);
usersUnq = unique(Tsub.User);
nUsers = height(Tsub);
fig3 = figure();
tlo = tiledlayout(fig3,'flow','TileSpacing','Compact','Padding','Compact');
for i = 1:nUsers
    ax = nexttile(tlo);
    idx = T.User==usersUnq(i);
    plot(ax,T.Views(idx),T.Votes(idx),'o','MarkerSize',4)
    title(ax,usersUnq(i),'Interpreter','none')
    axis(ax,'equal')
    axis(ax,'square')
    xlim(ax,xlim(ax)) % holds x limits 
    ylim(ax,ylim(ax)) % holds x limits 
    h=lsline(ax);
    h.Color = 'k';
    [r,p] = corr(T.Views(idx),T.Votes(idx));
    grid(ax,'on')
    if p>0.05
        h.LineStyle = '--';
    end
    text(ax,max(xlim(ax)),min(ylim(ax)),...
        sprintf('r = %.3f \np = %.3f ',r,p), ...
        'HorizontalAlignment', 'Right', 'VerticalAlignment','bottom', ...
        'FontSize', 8)
end
xlabel(tlo,'Views','FontSize',16)
ylabel(tlo,'Votes','FontSize',16)
title(tlo,'Correlation between view and votes','FontSize',18)
hold(ax,'on')
lh(1) = plot(ax,nan,nan,'k-','DisplayName','Significant');
lh(2) = plot(ax,nan,nan,'k--','DisplayName','Not significant correlation (p>0.05)');
leg = legend(ax,lh,'Orientation','horizontal');
leg.Layout.Tile = 'South';
subtitle(tlo,sprintf('Participants with more than %d submissions',minSubs-1))

%% Ratio of votes to views
% This section can be run indpendently when table T is available. 
% It produces a figure showing the ratio of votes to views. 
TSummary = groupsummary(T,'User','sum');
voteViewRatio = TSummary.sum_Votes ./ TSummary.sum_Views;
[sortedRatios, sortIdx] = sort(voteViewRatio);
sortedCats = reordercats(TSummary.User,string(TSummary.User(sortIdx)));
fig = figure(); 
ax = axes(fig);
bar(ax,sortedCats, voteViewRatio)
hold on
mu = mean(voteViewRatio);
yline(mu, 'm-', sprintf('Grand mean %.2f', mu),'LabelHorizontalAlignment','Left')
grid(ax,'on')
ylabel(ax,'#Votes/#Views')
title(ax,'Votes to views ratio')
yline(ax,1,'k--',['More votes than views',char(9650)],'LabelHorizontalAlignment','Left')

% Only show ratios > 0.1
cutoff = 0.1; 
catOrder = categories(sortedCats); 
xlim(ax,catOrder([find(sortedRatios>cutoff,1),numel(catOrder)]))
subtitle(ax,sprintf('Showing values >=%g',cutoff))
