Making version 7.3 MAT files directly from C++
The example code below shows how to write version 7.3 MAT files directly from C++ using the HDF5 library (libhdf5) and HighFive, a header-only C++ wrapper for libhdf5. Version 7.3 MAT files are HDF5-based, but contain a proprietary header in the first 512 bytes of the file.
The implementation performs three primary tasks:
First, it creates an HDF5 file with a 512-byte userblock. After data has been added into the file, the file is closed. Then a 128-byte header is written into the userblock so that the file is recognized by MATLAB as a valid version-7.3 MAT file. This is done in function “makeMatHeader”.
Second, MATLAB-specific metadata attributes are added to each dataset. Attributes such as “MATLAB_class” and “MATLAB_int_decode” inform MATLAB how each dataset should be interpreted.
Third, MATLAB-compatible complex datasets are created by overriding HighFive's default complex-number layout. HighFive uses the field names `r` and `i` by default, while MATLAB expects `real` and `imag`. A Highfive custom compound type is therefore registered for `std::complex<double>` using the MATLAB field names.
With these changes in place, C++ code can write scalar values, vectors, structs, complex arrays, and character arrays to a file that MATLAB can read as a version 7.3 MAT file.
#include <iostream>
#include <vector>
#include <complex>
#include <cstddef>
#include <fstream>
#include <string>
#include <cstdint>
#include <utility>
#include <bitset>
#include <highfive/highfive.hpp>
#include "hdf5.h"
// Modify the 512-byte userblock at the front of the HDF5 file to make it compatible with MATLAB's v7.3 MAT file format.
void makeMatHeader(std::string filename)
{
char header[512]; // MATLAB-style header for HDF5 file
memset(header, 0, sizeof(header)); // Initialize header to all zeros
// Example header content
snprintf(header, sizeof(header), "MATLAB 7.3 MAT-file, Platform: HDF5");
header[124] = 0;
header[125] = 2;
// I/M indicate little-endian format (Intel Mac/Windows)
header[126] = 'I';
header[127] = 'M';
// Write the header to the beginning of the file
std::ofstream outFile(filename, std::ios::binary | std::ios::in | std::ios::out);
outFile.seekp(0);
outFile.write(header, sizeof(header));
outFile.close();
}
// https://www.geeksforgeeks.org/dsa/inplace-m-x-n-size-matrix-transpose/
void MatrixInplaceTranspose(int *A, int rows, int cols)
{
// Moves elements in-place to achieve the transpose.
// A is a pointer to a 2D array, rows is the number of rows, and cols is the number of columns.
int size = rows*cols - 1;
int t; // holds element to be replaced, eventually becomes next element to move
int next; // location of 't' to be moved
int cycleBegin; // holds start of cycle
int i; // iterator
const int HASH_SIZE = 8192; // define a suitable hash size for the bitset. Must be at least as large as the number of elements in the matrix.
std::bitset<HASH_SIZE> b; // hash to mark moved elements. Must be large enough to cover all indices.
if (rows <= 0 || cols <= 0) {
throw std::invalid_argument("Matrix dimensions must be positive");
}
else if ((rows * cols) > HASH_SIZE)
{
throw std::invalid_argument("Matrix size exceeds hash size for in-place transpose. Increase the HASH_SIZE constant.");
}
b.reset();
b[0] = b[size] = 1;
i = 1; // Note that A[0] and A[size-1] won't move
while (i < size)
{
cycleBegin = i;
t = A[i];
do
{
// Input matrix [rows x cols]
// Output matrix [cols x rows]
// i_new = (i*rows)%(N-1)
next = (i*rows)%size;
std::swap(A[next], t);
b[i] = 1;
i = next;
}
while (i != cycleBegin);
// Get Next Move (what about querying random location?)
for (i = 1; (i < size) && b[i]; i++)
;
}
}
template <typename T>
std::vector<std::vector<T>> transpose(const std::vector<std::vector<T>>& matrix)
{
// Performs a nonconjugate transpose on a vector of vectors
// The input matrix is a vector of vectors, where each inner vector represents a row of the matrix.
// Handle empty matrix edge case
if (matrix.empty() || matrix[0].empty()) {
return {};
}
size_t rows = matrix.size();
size_t cols = matrix[0].size();
// Initialize the transposed matrix with flipped dimensions: cols x rows
std::vector<std::vector<T>> transposed(cols, std::vector<T>(rows));
for (size_t i = 0; i < rows; ++i) {
for (size_t j = 0; j < cols; ++j) {
transposed[j][i] = matrix[i][j];
}
}
return transposed;
}
// Creates a HighFive compound type for representing MATLAB-style complex numbers
// HighFive by default uses r/i but that is not compatible with MATLAB's complex number representation, which uses real/imag.
HighFive::CompoundType matlabComplexDouble () {
return {
{"real", HighFive::AtomicType<double>{}},
{"imag", HighFive::AtomicType<double>{}}
};
}
// Register the CompoundType to represent std::complex<double>
HIGHFIVE_REGISTER_TYPE(std::complex<double>, matlabComplexDouble);
int main()
{
const std::string filename = "test.mat";
// Needed for the complex number literal suffix 'i'
using namespace std::literals;
/*
* MATLAB vs C++ array layout
*
* MATLAB stores arrays in column-major order, meaning values in the same column are
* laid out next to each other in memory. Typical C++ containers such as nested std::vector and arrays
* are written in row-major order, where values in the same row are adjacent in memory.
*
* That difference matters when something such as a 2D dataset is exchanged from C++ to MATLAB. A 2x3
* matrix written from C++ in row-major order will be interpreted by MATLAB as a 3x2 matrix, transposed relative
* to the original C++ layout. The user will have to transpose the array to view the original C++ layout
* correctly.
*
* C++ developers need to be aware of the memory layout when
* exchanging multidimensional arrays with MATLAB. To maintain the structure,
* one will need to transpose the array before writing it to the mat file.
*/
// Test data
// 2x3 Array of complex double
std::vector<std::vector<std::complex<double>>> dataComplex = {{10.0 + 1.0i, 20.0 + 2.0i, 30.0 + 3.0i},
{40.0 + 4.0i, 50.0 + 5.0i, 60.0 + 6.0i}};
// 1x3 Vector of double
std::vector<double> dataDoubleVec = {1.1, 2.2, 3.3};
// 1x5 Array of integers
int dataIntArray[5] = {1, 2, 3, 4, 5};
// 2x4 Array of integers
int dataIntArray2x4[2][4] = {{1, 2, 3, 4},
{5, 6, 7, 8}};
int dataInt = 79;
double dataDouble = 3.14;
std::string dataString = "Hello, MATLAB!!!!!";
{
// Put the highfive related code into its own block so that the file gets closed when the file object is no longer in scope.
// Create a highfive file create property, get the underlying HDF5 property ID, and set a userblock size
HighFive::FileCreateProps fcp = HighFive::FileCreateProps::Empty();
hid_t fcpl_id = fcp.getId();
H5Pset_userblock(fcpl_id, 512);
HighFive::File file(filename, HighFive::File::Truncate, fcp);
// Storing a double to the file
// For something that is only a single value, must create a 1x1 dataspace
HighFive::DataSpace scalarDoubleSpace({1, 1});
// This creates a variable in the MATLAB workspace with the name "double_value"
HighFive::DataSet doubleField = file.createDataSet<double>("double_value", scalarDoubleSpace);
doubleField.write(dataDouble);
// Metadata for MATLAB compatibility
doubleField.createAttribute("MATLAB_class", std::string("double"));
// Storing an integer to the file
// For something that is only a single value, must create a 1x1 dataspace
HighFive::DataSpace scalarIntSpace({1, 1});
// This creates a variable in the MATLAB workspace with the name "int_value"
HighFive::DataSet intField = file.createDataSet<int>("int_value", scalarIntSpace);
intField.write(dataInt);
// Metadata for MATLAB compatibility
intField.createAttribute("MATLAB_class", std::string("int32"));
// Storing a C-style 1x5 array of integers to the file
// Since it is a single dimension, there is no need to move the data, just reinterpret it as a 5x1 row-major array.
// When Matlab imports it, it will perceive it as a 1x5 column-major array.
// This line casts the 1x5 array to a 5x1 array to match MATLAB's column-major order
int (*numArrayTrans5x1)[1] = reinterpret_cast<int (*)[1]>(dataIntArray);
HighFive::DataSpace intArray5x1Space({5, 1});
// This creates a variable in the MATLAB workspace with the name "int_array"
HighFive::DataSet intArrayField = file.createDataSet<int>("int_array", intArray5x1Space);
intArrayField.write(numArrayTrans5x1);
intArrayField.createAttribute("MATLAB_class", std::string("int32"));
// Storing a C-style 2x4 array of integers to the file
// For something that is a multi-dimensional array, we need to transpose the array and create a dataspace with the dimensions swapped
// so that the data is stored in column-major order.
MatrixInplaceTranspose((int*)dataIntArray2x4, 2, 4);
// After moving the values around, we need to cast the array with the new dimensions to match the new layout
// Cast the transposed 2x4 array to a 4x2 array to match MATLAB's column-major order
int (*numArrayTrans)[2] = reinterpret_cast<int (*)[2]>(dataIntArray2x4);
HighFive::DataSpace intArray2x4Space({4, 2});
// This creates a variable in the MATLAB workspace with the name "int_array_2x4"
HighFive::DataSet intArray2x4Field = file.createDataSet<int>("int_array_2x4", intArray2x4Space);
intArray2x4Field.write(numArrayTrans);
intArray2x4Field.createAttribute("MATLAB_class", std::string("int32"));
// Creating a Matlab struct (HDF5 group)
HighFive::Group my_struct = file.createGroup("my_struct");
my_struct.createAttribute("MATLAB_class", std::string("struct"));
// The only difference between storing data into a struct or as a normal variable in the MAT file is the
// the parent object you use when you do "createDataSet".
// file.createDataSet would create a normal variable, my_struct.createDataSet creates it within the "my_struct" struct.
// Storing a string to the struct so that it will be accessible as a character array in MATLAB
// For something that is a string, we create a dataspace with dimensions [string_length, 1] and save the character data accordingly
// We create a vector that has dataString.size() elements, each of which is a char vector of size 1 to store individual characters.
std::vector<std::vector<char>> text_bytes(dataString.size(), std::vector<char>(1));
// MATLAB expects character arrays to be a row vector so we reshape it accordingly since dimensions are swapped between C++ and MATLAB
for (int i = 0; i < dataString.size(); ++i) {
text_bytes[i][0] = dataString[i];
}
HighFive::DataSpace charSpace({dataString.size(), 1});
// uint16_t is required for MATLAB character arrays
HighFive::DataSet textField = my_struct.createDataSet<uint16_t>("text_value", charSpace);
textField.write(text_bytes);
// Metadata for MATLAB compatibility
textField.createAttribute("MATLAB_class", std::string("char"));
// Tell MATLAB to interpret the data as characters rather than integers
textField.createAttribute("MATLAB_int_decode", 2);
// Storing a vector to the struct
// In order to transpose the vector correctly, we first wrap it in another vector to make it a 2D array.
std::vector<std::vector<double>> transposableDoubleVec = { dataDoubleVec };
// For vectors, we let HighFive infer the dataspace from the data itself
// Transpose the vector to match MATLAB's column-major order
HighFive::DataSet doubleVectorField = my_struct.createDataSet("double_vector", transpose(transposableDoubleVec));
// Metadata for MATLAB compatibility
doubleVectorField.createAttribute("MATLAB_class", std::string("double"));
// Storing a complex (and multi-dimensional) vector to the struct
// For multi-dimensional vectors, we need to perform a noncojugate transpose on the array to match
// MATLAB's column-major order so the data layout is consistent between C++ and MATLAB.
// For vectors, we let HighFive infer the dataspace from the data itself
HighFive::DataSet complexField = my_struct.createDataSet("complex_vector", transpose(dataComplex));
// Metadata for MATLAB compatibility
complexField.createAttribute("MATLAB_class", std::string("complex"));
}
// Finalize the MATLAB-compatible HDF5 file by writing the MATLAB header into the userblock
makeMatHeader(filename);
return 0;
}
Tags
