Code covered by the BSD License  

Highlights from
Active Shape Model (ASM) and Active Appearance Model (AAM)

image thumbnail

Active Shape Model (ASM) and Active Appearance Model (AAM)

by

Dirk-Jan Kroon (view profile)

 

16 Feb 2010 (Updated )

Cootes 2D/3D Active Shape & Appearance Model for automatic image object segmentation and recognition

AAM_ApplyModel2D(ItestLarge,tformLarge,Data,options)
function AAM_ApplyModel2D(ItestLarge,tformLarge,Data,options)

% We start at the coarse scale
scale=1; scaling=2^(-(scale-1));

% Transform the coordinates to match the coarse scale
tform=tformLarge;
%tform.offsetv=(tform.offsetv-0.5)*scaling+0.5;
tform.offsetv=tform.offsetv*scaling;

% Get the PCA model for this scale
ShapeAppearanceData=Data{scale}.ShapeAppearanceData;
ShapeData=Data{scale}.ShapeData;
AppearanceData=Data{scale}.AppearanceData;

% Use the mean ShapeAppearance parameters to go get an initial contour
b = ShapeAppearanceData.b_mean;
b1 = b(1:(length(ShapeAppearanceData.Ws)));
b1= inv(ShapeAppearanceData.Ws)*b1;
% Initial (mean) aligned coordinates
x = ShapeData.x_mean + ShapeData.Evectors*b1;
% The real image coordinates
pos=[x(1:end/2) x(end/2+1:end)];
pos=AAM_align_data_inverse2D(pos,tform);



% Loop through the 4 image size scales
for scale=options.nscales:-1:1
    % Get the PCA model for this scale
    R=Data{scale}.R;
    ShapeAppearanceData=Data{scale}.ShapeAppearanceData;
    ShapeData=Data{scale}.ShapeData;
    AppearanceData=Data{scale}.AppearanceData;
    TrainingData=Data{scale}.TrainingData;
    % The image scaling of the scale-itteration
    scaling=2^(-(scale-1));

    % Transform the image and coordinates offset, to the cuurent scale
    Itest=imresize(ItestLarge,scaling);
    
    pos=(pos-0.5)*scaling+0.5; 

    % From real image coordinates to -> algined coordinates
    [pos_align, tform]=AAM_align_data2D(pos, ShapeData.MeanVertices );
    x=[pos_align(:,1);pos_align(:,2)];
    
    % Start a new figure
    % show current test image, and initial contour
    figure, imshow(Itest); hold on; plot(pos(:,2),pos(:,1),'r.'); h=plot(1,1); 
   
    % Sample the image intensities 
    g=AAM_Appearance2Vector2D(Itest,pos, AppearanceData.base_points, AppearanceData.ObjectPixels,ShapeData.TextureSize,ShapeData.Tri);
    g=AAM_NormalizeAppearance2D(g);

    % Go from image intesities and contour to ShapeAppearance parameters
    b1 = ShapeAppearanceData.Ws * ShapeData.Evectors' * (x-ShapeData.x_mean);
    b2 = AppearanceData.Evectors' * (g-AppearanceData.g_mean);
    b = [b1;b2];
    cin = ShapeAppearanceData.Evectors'*(b -ShapeAppearanceData.b_mean);
    x2=x;
    maxc=options.m*sqrt(ShapeAppearanceData.Evalues);
    c = lsqnonlin(@(x)costc(x,x2,ShapeAppearanceData,ShapeData),cin,-maxc,maxc,optimset('Display','off','MaxIter',10));
    
    %  Storage of ShapeAppeanance parameters, pose parameters, last
    %  error between model and intensities. Used if old location
    %  had a smaller intensity error.
    c_old=c; tform_old=tform; Eold=inf; 
    
    % Starting step size
    w=1; 
    
    % Search Itterations
    for i=(1:options.nsearch)
        % Go from ShapeAppearance Parameters to aligned shape coordinates
        b = ShapeAppearanceData.b_mean + ShapeAppearanceData.Evectors*c;
        b1 = b(1:(length(ShapeAppearanceData.Ws)));
        b1= inv(ShapeAppearanceData.Ws)*b1;
        x = ShapeData.x_mean + ShapeData.Evectors*b1;
        
        % From aligned coordinates to real image coordinates
        pos=AAM_align_data_inverse2D([x(1:end/2) x(end/2+1:end)],tform);
    
        % Sample the intensities
        g=AAM_Appearance2Vector2D(Itest,pos, AppearanceData.base_points, AppearanceData.ObjectPixels,ShapeData.TextureSize,ShapeData.Tri);
        g=AAM_NormalizeAppearance2D(g);

        % Go from intensities and shape back to ShapeAppearance Parameters
        b1 = ShapeAppearanceData.Ws * ShapeData.Evectors' * (x-ShapeData.x_mean);
        b2 = AppearanceData.Evectors' * (g-AppearanceData.g_mean);
        b = [b1;b2];
        c2 = ShapeAppearanceData.Evectors'*(b -ShapeAppearanceData.b_mean);
                
        % Go from ShapeAppearance Parameters back to model intensities
        b = ShapeAppearanceData.b_mean + ShapeAppearanceData.Evectors*c2;
        b2 = b(size(ShapeAppearanceData.Ws,1)+1:end);
        g_model = AppearanceData.g_mean + AppearanceData.Evectors*b2;

        % Difference between model and real image intensities
        E=sum((g-g_model).^2);

        % Go back to the old location of the previous itteration, if the
        % error was lower.
        if(E>Eold)
            % Not always go back if the error becomes higher, sometimes
            % stay on the higher error (like in simulated annealing)
            % Try a smaller stepsize
                w=w*0.9;
                c=c_old; tform=tform_old;
        else
            w=w*1.1;
            Eold=E;
        end
   

        % Store model /pose parameters for next itteration
        c_old=c;
        tform_old=tform;
        
        % Calculate the needed model parameter update using the 
        % search model matrix
        c_diff=R*(g-g_model);
        % Update the ShapeApppearance Parameters
        c=c+c_diff(1:end-4).*sqrt(ShapeAppearanceData.Evalues)*w;
        % Update the Pose parameters
        tform.offsetv(1)=tform.offsetv(1)+c_diff(end-3)*w;
        tform.offsetv(2)=tform.offsetv(2)+c_diff(end-2)*w;  
        tform.offsetsx=tform.offsetsx+c_diff(end-1)*w;
        tform.offsetsy=tform.offsetsy+c_diff(end-0)*w;
        
        % Stay within 3 (m) standard deviations               
        maxc=options.m*sqrt(ShapeAppearanceData.Evalues);
        c=max(min(c,maxc),-maxc);
        
        % Show the current contour
        delete(h); h=plot(pos(:,2),pos(:,1),'b.'); 
        % Pause for a moment
        pause(0.1);
    end
    
    pos=(pos-0.5)/scaling+0.5; 
end
if(E>Eold), c=c_old; tform=tform_old; end   

% Go from ShapeAppearance Parameters to Shape and Appearance Parameters
b = ShapeAppearanceData.b_mean + ShapeAppearanceData.Evectors*c;
b1 = b(1:(length(ShapeAppearanceData.Ws)));
b1= inv(ShapeAppearanceData.Ws)*b1;
b2 = b(size(ShapeAppearanceData.Ws,1)+1:end);
g_model = AppearanceData.g_mean + AppearanceData.Evectors*b2;
x = ShapeData.x_mean + ShapeData.Evectors*b1;
        
% From aligned coordinates to real image coordinates
pos=AAM_align_data_inverse2D([x(1:end/2) x(end/2+1:end)],tform);

% Sample the intensities
g=AAM_Appearance2Vector2D(Itest,pos, AppearanceData.base_points, AppearanceData.ObjectPixels,ShapeData.TextureSize,ShapeData.Tri);
c1=std(g(:)); c2=mean(g(:));
g=AAM_NormalizeAppearance2D(g);

E_init=sum( (g(:)-g_model(:)).^2);
disp(E_init)


% Show difference between modeled and real intensities of the test image
% object, in the mean shape
figure, 
I_texture=AAM_Vector2Appearance2D(g_model*c1+c2,AppearanceData.ObjectPixels,ShapeData.TextureSize);
I_texture2=AAM_Vector2Appearance2D(g*c1+c2,AppearanceData.ObjectPixels,ShapeData.TextureSize);

subplot(1,2,1), imshow(I_texture); title('model intensities');
subplot(1,2,2), imshow(I_texture2); title('real intensities');

% Show the modeled image
figure,
I_texture=AAM_Vector2Appearance2D(g_model*c1+c2,AppearanceData.ObjectPixels,ShapeData.TextureSize);

% The Piece Wise linear transformation below, only works if contour points are
% unique, and the number of fold-over triangles is small
input_points= AppearanceData.base_points;
base_points=pos;

% Remove control points which give folded over triangles with cp2tform
xy=[input_points(:,2) input_points(:,1)];
uv=[base_points(:,2) base_points(:,1)];
[xy uv]=PreProcessCp2tform(xy,uv);
trans_prj = cp2tform(xy,uv,'piecewise linear');

% Transform the image into the default texture image
ImodelLarge = imtransform(I_texture,trans_prj,'Xdata',[1 size(ItestLarge,2)],'YData',[1 size(ItestLarge,1)],'XYscale',1);
ImodelLarge(isnan(ImodelLarge))=0;
imshow(ImodelLarge); title('The model shown on the found location');

IsegmentedLarge= drawObject(base_points,[size(ItestLarge,1) size(ItestLarge,2)],Data{1}.ShapeData.Lines);
figure, imshow(IsegmentedLarge), title('Area contour');


function E=costc(c,x2,ShapeAppearanceData,ShapeData)
% Probably also solve able with lsqlin (linear bound solve)...

% Go from ShapeAppearance Parameters to aligned shape coordinates
b = ShapeAppearanceData.b_mean + ShapeAppearanceData.Evectors*c;
b1 = b(1:(length(ShapeAppearanceData.Ws)));
b1= inv(ShapeAppearanceData.Ws)*b1;
x = ShapeData.x_mean + ShapeData.Evectors*b1;
E=x(:)-x2(:);


Contact us