image thumbnail

Easily store and retrieve subsets on big matrices on disk

by

 

Stores columns of a matrix one at a time in hard disk and permits to easily recover subsets of it.

SimpleSequence
% Sequence class
% ---------------------
% Permits to store big sequences of real matrices into the hard disk
% and to retrieve the data easily.
%
% Example Usage : small test
%-----------------------------
% M = 1024;
% N  = 4096; 
% test_data = single(randn(M,N));
% 
% file = SimpleSequence('filename',M); %create object
% for n = 1:N
%   file.append(test_data(:,n)); % append data, one at a time
% end
% file.close(); %close object 
%
% %Retrieve all data
% data = file.get(Inf,Inf); %retrieve all data
% fprintf('Error is %f\n',norm(test_data - data))%

% % Retrieve a subsampling of the lines of data
% data_sub_lines = file.get(1:3:M,Inf);
%
% % Retrieve a subsampling of the columns of data
% data_sub_cols = file.get(Inf,1:3:N); 
classdef SimpleSequence<handle
    properties
        filename,M,N
    end
    properties (Access=private)
        write_handle
    end
    
    methods
        function self = SimpleSequence(filename,M)
            % Creates the object where each data is Mx1
            self.filename = filename;
            self.M = M;
            self.N = 0;
            self.filename =[filename '.avi'];
            self.write_handle = [];
        end
        
        function delete(self)
            %deletes object
            self.close()
        end
        
        function reset(self)
            %creates a new file (erases previous one)
            try
                self.write_handle = VideoWriter(self.filename,'Uncompressed AVI');
                self.write_handle.open();
            catch 
                error('Error while creating file. Aborting')
            end
            disp('Opening file : success.');
            self.N = 0;
        end
        
        function close(self)
            %closes currently opened file
            try
                if ~isempty(self.write_handle)
                    self.write_handle.close()
                end
            catch
                error('Closing file : error.');
            end
            disp('Closing file : success.');
            self.write_handle = [];
        end
        
        function append(self, data)
            %Append data to the sequence. numel(data) must be
            %self.Mm*self.Mn
            if isempty(self.write_handle)
                self.reset()
            end
            zero_append = 3- rem(self.M,3);
            data_vec_single = single([data(:); single(zeros(zero_append,1))]);
            data_vec_uint8 = typecast(data_vec_single,'uint8');
            L = length(data_vec_uint8);
            self.write_handle.writeVideo(reshape(data_vec_uint8,L/3,1,3));
            self.N = self.N + 1;
        end 
        
        function shape = size(self)
            %Returns the size of the data
            shape = [self.M,self.N];
        end
        
        function H = get(self, lines,frames)
            %returns a subset of the data. lines is a vector indicating the
            %lines to selec, same thing for the columns and frames.
            % Inf instead of a vector for one of them selects everything
            if ~isempty(self.write_handle)
                error('Cannot retrieve data from an open file. Call close() first');
            end
            try
                read_handle = VideoReader(self.filename);
            catch             
                error('Error while retrieving data. Aborting.');
            end
             
            if lines == Inf
                lines = 1:self.M;
            end
            if frames == Inf
                frames = 1:self.N;
            end
            H = zeros(length(lines),length(frames));
            zero_append = 3- rem(self.M,3);
            disp('retrieving data....')
            for iframe = 1:length(frames)
                raw_data = read_handle.read(frames(iframe));
                data_single = typecast(raw_data(:),'single');
                frame_data = data_single(1:end-zero_append);
                H(:,iframe) = double(frame_data(lines));
            end
        end
    end
end

Contact us