I've done it (I think)
ft=fittype('a*x^5+b*x^3+26*x');
fitobj,~,~,~]=fit(x,y,ft) %[fitobj, goodness, output, convmsg]=fit(x,y,ft)
coeff1=fitobj.a;
coeff2=fitobj.b;
yfit=coeff1*x.^5+coeff2*x.^3+26*x; %Remember dot notation
r=y-yfit; %Calc residuals