A set of MATLAB functions for evaluating generalization performance in binary classification.
acc_cmp2(C1,C2)
% Tests the null hypothesis that two classifiers that were tested on
% independent datasets have the same prediction accuracy on unseen data.
%
% This function is to be used to compare the accuracies obtained on
% independent datasets. In order to compare two classifiers used on the
% same dataset you have to do a paired comparison using ACC_CMP1.
%
% Usage:
% p = acc_cmp2(C1,C2)
%
% Arguments:
% C1 - 2x2 confusion matrix of classification outcomes
% C2 - 2x2 confusion matrix of classification outcomes
%
% See also:
% acc_cmp1
% Kay H. Brodersen, ETH Zurich, Switzerland
% http://people.inf.ethz.ch/bkay/
% $Id: acc_cmp2.m 8247 2010-10-22 13:34:28Z bkay $
% -------------------------------------------------------------------------
function p = acc_cmp2(C1,C2)
% The size \alpha Wald test is to reject H_0 when |W|>z_\alpha/2, where
% W = \frac{\delta_hat - 0}{se_hat}
% and
% \delta_hat = p1 - p2
% Prepare test statistic
p1 = acc_mode(C1);
p2 = acc_mode(C2);
m = sum(sum(C1));
n = sum(sum(C2));
W = (p1-p2) / sqrt(p1*(1-p1)/m + p2*(1-p2)/n);
% Return p value of Wald test
p = 1-normcdf(W,0,1);
end