perf=sudukubench(solver,nTest)

function perf=sudukubench(solver,nTest)
% SUDOKUBENCH   Benchmark of Sudoku solvers
%
% perf = sudokubench(solver,nTest) returns performance of the specified
% sudoku solver against designated 3-level benchmark puzzles.
%
% Input:    solver - function handle of sudoku solver. The interface of
%                    sudoku solver should be solution = solver(puzzle).
%           nTest - number of puzzles for each level, default = 100.
% Output:   perf - structure cell array of performance measures for 3 levels
%           perf.error: the ratio of the number of incorrect solutions
%           orver total number of puzzles in different levels.
%           perf.time: the ratio of the average solution time of different
%           levels against the benchark solver, YASS, i.e. if perf.time <
%           1, the specified solver is faster than YASS, otherwise,
%           the specified solver is slower than YASS.
%           
% Examples:
% 
% Example 1: against itself with 100 x 3 puzzles 
%{
tic,perf=sudukubench(@yass);toc % about 27 seconds
perf{:}
%}
% ans = 
% 
%     error: 0
%      time: 0.9579
% 
% 
% ans = 
% 
%     error: 0
%      time: 1.0045
% 
% 
% ans = 
% 
%     error: 0
%      time: 1.0161
%
% Example 2: Fast Sudoku Solver by Michael Kleder (File Exchange ID 13324)
% You have to download the code before run the test.
%{
perf=sudukubench(@sudoku,5);
perf{:}
%}
%
% Example 3: MATLAB Sudoku Solver by Geoff Boynton (File Exchange ID 8083)
% Warning: It is very slow solver and may take very long time.
% You have to download the code before run the test.
%{
perf=sudukubench(@sodoku,1);
perf{:}
%}
%
% Example 4: Sudoku: ASP.NET/MATLAB by John Kotwicki (File Exchange ID 12096)
% Warning: Extreamly slow solve, it may take hours.
% You have to download the code before run the test.
%{
perf=sudukubench(@solve_sudoku,1);
perf{:}
%}
%
% By Yi Cao at Cranfield University on 25th Feb 2008
%

% Input and output check
error(nargchk(1,2,nargin));
error(nargoutchk(0,1,nargout));
if nargin<2
    nTest=100;
end

load supersudokulib sudokulib Level
nV=sum(Level);
perf=cell(3,1);
A=zeros(9);
for L=1:3
    err=0;
    et1=0;
    et2=0;
    [dum,idx]=sort(rand(nV(L),1));
    I=find(Level(:,L));
    idx=I(idx(1:nTest));
    for k=idx(:)'
        A(:)=sudokulib(k,:);
        t0=cputime;
        B1=yass(A);
        t1=cputime;
        try
            B2=solver(A);
        catch
            err=err+1;
            et2=et2+cputime-t1;
            continue
        end
        et2=et2+cputime-t1;
        et1=et1+t1-t0;
        if ~isequal(B1,B2)
            err=err+1;
        end
    end
    p.error=err/nTest;
    p.time=et2/et1;
    perf{L}=p;
end