Loading include/dca/io/hdf5/hdf5_reader.hpp +154 −178 Original line number Diff line number Diff line Loading @@ -34,7 +34,7 @@ public: typedef H5::H5File file_type; // In: verbose. If true, the reader outputs a short log whenever it is executed. HDF5Reader(bool verbose = true) : my_file(NULL), my_paths(0), verbose_(verbose) {} HDF5Reader(bool verbose = true) : verbose_(verbose) {} ~HDF5Reader(); Loading @@ -49,10 +49,10 @@ public: void close_file(); void open_group(std::string name) { my_paths.push_back(name); paths_.push_back(name); } void close_group() { my_paths.pop_back(); paths_.pop_back(); } std::string get_path(); Loading @@ -65,11 +65,14 @@ public: template <typename scalartype> bool execute(std::string name, scalartype& value); template <typename scalar_type> bool execute(std::string name, std::vector<scalar_type>& value); template <typename Scalar> bool execute(std::string name, std::vector<Scalar>& value); template <typename scalar_type> bool execute(std::string name, std::vector<std::complex<scalar_type>>& value); template <typename Scalar> bool execute(std::string name, std::vector<std::complex<Scalar>>& value); template <typename Scalar> bool execute(std::string name, std::vector<std::vector<Scalar>>& value); bool execute(std::string name, std::string& value); Loading @@ -87,30 +90,33 @@ public: template <typename scalartype, typename domain_type> bool execute(std::string name, func::function<scalartype, domain_type>& f); template <typename scalar_type> bool execute(std::string name, dca::linalg::Vector<scalar_type, dca::linalg::CPU>& A); template <typename Scalar> bool execute(std::string name, dca::linalg::Vector<Scalar, dca::linalg::CPU>& A); template <typename scalar_type> bool execute(std::string name, dca::linalg::Vector<std::complex<scalar_type>, dca::linalg::CPU>& A); template <typename Scalar> bool execute(std::string name, dca::linalg::Vector<std::complex<Scalar>, dca::linalg::CPU>& A); template <typename scalar_type> bool execute(std::string name, dca::linalg::Matrix<scalar_type, dca::linalg::CPU>& A); template <typename Scalar> bool execute(std::string name, dca::linalg::Matrix<Scalar, dca::linalg::CPU>& A); template <typename scalar_type> bool execute(std::string name, dca::linalg::Matrix<std::complex<scalar_type>, dca::linalg::CPU>& A); template <typename Scalar> bool execute(std::string name, dca::linalg::Matrix<std::complex<Scalar>, dca::linalg::CPU>& A); template <typename scalar_type> bool execute(dca::linalg::Matrix<scalar_type, dca::linalg::CPU>& A); template <typename Scalar> bool execute(dca::linalg::Matrix<Scalar, dca::linalg::CPU>& A); bool execute(std::string name, io::Buffer& buff) { return execute(name, static_cast<io::Buffer::Container&>(buff)); } private: bool fexists(const char* filename); bool exists(const std::string& name) const; void read(const std::string& name, H5::PredType type, void* data) const; std::vector<hsize_t> readSize(const std::string& name) const; H5::H5File* my_file; std::vector<std::string> my_paths; std::unique_ptr<H5::H5File> file_; std::vector<std::string> paths_; bool verbose_; }; Loading @@ -123,69 +129,90 @@ void HDF5Reader::from_file(arbitrary_struct_t& arbitrary_struct, std::string fil reader_obj.close_file(); } template <typename scalar_type> bool HDF5Reader::execute(std::string name, scalar_type& value) { template <typename Scalar> bool HDF5Reader::execute(std::string name, Scalar& value) { std::string full_name = get_path() + "/" + name; try { H5::DataSet dataset = my_file->openDataSet(full_name.c_str()); H5::DataSpace dataspace = dataset.getSpace(); H5Dread(dataset.getId(), HDF5_TYPE<scalar_type>::get(), dataspace.getId(), H5S_ALL, H5P_DEFAULT, &value); if (!exists(full_name)) { return false; } read(full_name, HDF5_TYPE<Scalar>::get_PredType(), &value); return true; } catch (...) { std::cout << "\n\n\t the variable (" + name + ") does not exist in path : " + get_path() + "\n\n"; template <typename Scalar> bool HDF5Reader::execute(std::string name, std::vector<Scalar>& value) { std::string full_name = get_path() + "/" + name; if (!exists(full_name)) { return false; } auto dims = readSize(full_name); assert(dims.size() == 1); value.resize(dims.at(0)); read(full_name, HDF5_TYPE<Scalar>::get_PredType(), value.data()); return true; } template <typename scalar_type> bool HDF5Reader::execute(std::string name, std::vector<scalar_type>& value) { template <typename Scalar> bool HDF5Reader::execute(std::string name, std::vector<std::complex<Scalar>>& value) { std::string full_name = get_path() + "/" + name; try { H5::DataSet dataset = my_file->openDataSet(full_name.c_str()); value.resize(dataset.getInMemDataSize() / sizeof(scalar_type)); if (!exists(full_name)) { return false; } H5::DataSpace dataspace = dataset.getSpace(); auto dims = readSize(full_name); assert(dims.size() == 2); value.resize(dims.at(0)); H5Dread(dataset.getId(), HDF5_TYPE<scalar_type>::get(), dataspace.getId(), H5S_ALL, H5P_DEFAULT, &value[0]); read(full_name, HDF5_TYPE<Scalar>::get_PredType(), value.data()); return true; } catch (...) { std::cout << "\n\n\t the variable (" + name + ") does not exist in path : " + get_path() + "\n\n"; template <typename Scalar> bool HDF5Reader::execute(std::string name, std::vector<std::vector<Scalar>>& value) { std::string full_name = get_path() + "/" + name; if (!exists(full_name)) { return false; } } template <typename scalar_type> bool HDF5Reader::execute(std::string name, std::vector<std::complex<scalar_type>>& value) { std::string full_name = get_path() + "/" + name; const bool equal_size = !exists(full_name + "/data"); try { H5::DataSet dataset = my_file->openDataSet(full_name.c_str()); if (equal_size) { auto dims = readSize(full_name); assert(dims.size() == 2); std::vector<Scalar> linearized(dims[0] * dims[1]); value.resize(dataset.getInMemDataSize() / sizeof(std::complex<scalar_type>)); read(full_name, HDF5_TYPE<Scalar>::get_PredType(), linearized.data()); value.resize(dims[0]); const Scalar* read_location = linearized.data(); for (auto& v : value) { v.resize(dims[1]); std::copy_n(read_location, dims[1], v.data()); read_location += dims[1]; } } else { open_group(name); H5::DataSpace dataspace = dataset.getSpace(); int size = -1; execute("size", size); value.resize(size); H5Dread(dataset.getId(), HDF5_TYPE<scalar_type>::get(), dataspace.getId(), H5S_ALL, H5P_DEFAULT, &value[0]); return true; open_group("data"); for (int i = 0; i < value.size(); ++i) { execute("row_" + std::to_string(i), value[i]); } catch (...) { std::cout << "\n\n\t the variable (" + name + ") does not exist in path : " + get_path() + "\n\n"; return false; close_group(); close_group(); } return true; } template <typename scalartype, typename domain_type> Loading @@ -195,157 +222,106 @@ bool HDF5Reader::execute(func::function<scalartype, domain_type>& f) { template <typename scalartype, typename domain_type> bool HDF5Reader::execute(std::string name, func::function<scalartype, domain_type>& f) { std::cout << "\n\tstart reading function : " << name; open_group(name); bool success = true; try { std::string full_name = get_path() + "/data"; H5::DataSet dataset = my_file->openDataSet(full_name.c_str()); H5::DataSpace dataspace = dataset.getSpace(); std::string full_name = get_path() + "/" + name; H5Dread(dataset.getId(), HDF5_TYPE<scalartype>::get(), dataspace.getId(), H5S_ALL, H5P_DEFAULT, &f(0)); } catch (const H5::FileIException& err) { if (!exists(full_name)) { std::cout << "\n\n\t the function (" + name + ") does not exist in path : " + get_path() + "\n\n"; success = false; } close_group(); return success; return false; } template <typename scalar_type> bool HDF5Reader::execute(std::string name, dca::linalg::Vector<scalar_type, dca::linalg::CPU>& V) { bool success = true; try { std::string full_name = get_path() + "/" + name; std::cout << "\n\tstart reading function : " << name; H5::DataSet dataset = my_file->openDataSet(full_name.c_str()); read(full_name + "/data", HDF5_TYPE<scalartype>::get_PredType(), f.values()); V.resize(dataset.getInMemDataSize() / sizeof(scalar_type)); return true; } H5::DataSpace dataspace = dataset.getSpace(); template <typename Scalar> bool HDF5Reader::execute(std::string name, dca::linalg::Vector<Scalar, dca::linalg::CPU>& V) { std::string full_name = get_path() + "/" + name; if (!exists(full_name)) { return false; } H5Dread(dataset.getId(), HDF5_TYPE<scalar_type>::get(), dataspace.getId(), H5S_ALL, H5P_DEFAULT, &V[0]); auto dims = readSize(full_name); assert(dims.size() == 1); V.resize(dims.at(0)); V.set_name(name); } catch (const H5::FileIException& err) { std::cout << "\n\n\t the vector (" + name + ") does not exist in path : " + get_path() + "\n\n"; success = false; } read(full_name, HDF5_TYPE<Scalar>::get_PredType(), V.ptr()); return success; return true; } template <typename scalar_type> template <typename Scalar> bool HDF5Reader::execute(std::string name, dca::linalg::Vector<std::complex<scalar_type>, dca::linalg::CPU>& V) { bool success = true; try { dca::linalg::Vector<std::complex<Scalar>, dca::linalg::CPU>& V) { std::string full_name = get_path() + "/" + name; if (!exists(full_name)) { return false; } H5::DataSet dataset = my_file->openDataSet(full_name.c_str()); V.resize(dataset.getInMemDataSize() / sizeof(std::complex<scalar_type>)); H5::DataSpace dataspace = dataset.getSpace(); H5Dread(dataset.getId(), HDF5_TYPE<scalar_type>::get(), dataspace.getId(), H5S_ALL, H5P_DEFAULT, &V[0]); auto dims = readSize(full_name); assert(dims.size() == 2); V.resize(dims.at(0)); V.set_name(name); } catch (const H5::FileIException& err) { std::cout << "\n\n\t the vector (" + name + ") does not exist in path : " + get_path() + "\n\n"; success = false; } read(full_name, HDF5_TYPE<Scalar>::get_PredType(), V.ptr()); return success; return true; } template <typename scalar_type> bool HDF5Reader::execute(std::string name, dca::linalg::Matrix<scalar_type, dca::linalg::CPU>& A) { bool success = true; try { template <typename Scalar> bool HDF5Reader::execute(std::string name, dca::linalg::Matrix<Scalar, dca::linalg::CPU>& A) { std::string full_name = get_path() + "/" + name; if (!exists(full_name)) { return false; } H5::DataSet dataset = my_file->openDataSet(full_name.c_str()); H5::DataSpace dataspace = dataset.getSpace(); std::array<hsize_t, 2> dims; dataspace.getSimpleExtentDims(dims.data(), nullptr); auto dims = readSize(full_name); assert(dims.size() == 2); std::vector<scalar_type> linearized(dims[0] * dims[1]); dataset.read(linearized.data(), HDF5_TYPE<scalar_type>::get_PredType(), dataspace); std::vector<Scalar> linearized(dims[0] * dims[1]); read(full_name, HDF5_TYPE<Scalar>::get_PredType(), linearized.data()); // HDF5 is column major, while Matrix is not major. // HDF5 is column major, while Matrix is row major. A.resizeNoCopy(std::make_pair(dims[0], dims[1])); unsigned linindex = 0; for (int i = 0; i < A.nrRows(); ++i) { for (int i = 0, linindex = 0; i < A.nrRows(); ++i) { for (int j = 0; j < A.nrCols(); ++j) A(i, j) = linearized[linindex++]; } A.set_name(name); } catch (const H5::FileIException& err) { std::cout << "\n\n\t the function (" + name + ") does not exist in path : " + get_path() + "\n\n"; success = false; } return success; return true; } template <typename scalar_type> template <typename Scalar> bool HDF5Reader::execute(std::string name, dca::linalg::Matrix<std::complex<scalar_type>, dca::linalg::CPU>& A) { bool success = true; try { dca::linalg::Matrix<std::complex<Scalar>, dca::linalg::CPU>& A) { std::string full_name = get_path() + "/" + name; if (!exists(full_name)) { return false; } H5::DataSet dataset = my_file->openDataSet(full_name.c_str()); H5::DataSpace dataspace = dataset.getSpace(); std::array<hsize_t, 3> dims; dataspace.getSimpleExtentDims(dims.data(), nullptr); A.resizeNoCopy(std::make_pair(dims[1], dims[2])); std::vector<std::complex<scalar_type>> linearized(A.nrRows() * A.nrCols()); auto dims = readSize(full_name); assert(dims.size() == 3); dataset.read(linearized.data(), HDF5_TYPE<scalar_type>::get_PredType(), dataspace); std::vector<std::complex<Scalar>> linearized(dims[0] * dims[1]); read(full_name, HDF5_TYPE<Scalar>::get_PredType(), linearized.data()); // HDF5 is column major, while Matrix is not major. unsigned linindex = 0; for (int i = 0; i < A.nrRows(); ++i) for (int j = 0; j < A.nrCols(); ++j) { // HDF5 is column major, while Matrix is row major. A.resizeNoCopy(std::make_pair(dims[0], dims[1])); for (int i = 0, linindex = 0; i < A.nrRows(); ++i) { for (int j = 0; j < A.nrCols(); ++j) A(i, j) = linearized[linindex++]; } A.set_name(name); } catch (const H5::FileIException& err) { std::cout << "\n\n\t the function (" + name + ") does not exist in path : " + get_path() + "\n\n"; success = false; } return success; return true; } template <typename scalar_type> bool HDF5Reader::execute(dca::linalg::Matrix<scalar_type, dca::linalg::CPU>& A) { template <typename Scalar> bool HDF5Reader::execute(dca::linalg::Matrix<Scalar, dca::linalg::CPU>& A) { return execute(A.get_name(), A); } Loading include/dca/io/hdf5/hdf5_writer.hpp +17 −32 Original line number Diff line number Diff line Loading @@ -145,7 +145,7 @@ public: private: bool fexists(const char* filename); bool exists(const std::string& name); bool exists(const std::string& name) const; void write(const std::string& name, const std::vector<hsize_t>& size, H5::PredType type, const void* data); Loading Loading @@ -203,7 +203,7 @@ void HDF5Writer::execute(const std::string& name, const std::vector<std::complex<scalar_type>>& value) { if (value.size() > 0) { std::string full_name = get_path() + "/" + name; std::vector<hsize_t> dims{2, value.size()}; std::vector<hsize_t> dims{value.size(), 2}; write(full_name, dims, HDF5_TYPE<scalar_type>::get_PredType(), value.data()); } } Loading @@ -226,21 +226,27 @@ void HDF5Writer::execute(const std::string& name, const std::vector<std::vector< std::vector<hsize_t> dims{value.size(), cols}; std::vector<scalar_type> linearized(dims[0] * dims[1]); std::size_t linindex = 0; for (std::size_t i = 0; i < value.size(); ++i) for (std::size_t j = 0; j < cols; ++j, ++linindex) linearized[linindex] = value[i][j]; for (std::size_t i = 0, linindex = 0; i < value.size(); ++i) for (std::size_t j = 0; j < cols; ++j) linearized[linindex++] = value[i][j]; write(full_name, dims, HDF5_TYPE<scalar_type>::get_PredType(), value.data()); write(full_name, dims, HDF5_TYPE<scalar_type>::get_PredType(), linearized.data()); } else { open_group(full_name); execute("size", value.size()); open_group("data"); std::vector<hsize_t> dims(1); for (std::size_t i = 0; i < value.size(); ++i) { const std::string new_name = full_name + "/row_" + std::to_string(i); const std::string new_name = full_name + "/data/row_" + std::to_string(i); dims[0] = value[i].size(); write(new_name, dims, HDF5_TYPE<scalar_type>::get_PredType(), value[i].data()); } close_group(); close_group(); } } Loading @@ -252,30 +258,9 @@ void HDF5Writer::execute(const std::string& name, const std::vector<std::array<S return; std::array<hsize_t, 2> dims{value.size(), n}; H5::DataSet* dataset = nullptr; H5::DataSpace* dataspace = nullptr; std::string full_name = get_path() + "/" + name; dataspace = new H5::DataSpace(2, dims.data()); dataset = new H5::DataSet( file_->createDataSet(full_name.c_str(), HDF5_TYPE<Scalar>::get_PredType(), *dataspace)); Scalar* tmp = new Scalar[dims[0] * dims[1]]; // hdf5 has row-major ordering! // for (hsize_t i = 0; i < dims[0]; i++) // for (hsize_t j = 0; j < dims[1]; j++) // tmp[i * dims[1] + j] = value[i][j]; H5Dwrite(dataset->getId(), HDF5_TYPE<Scalar>::get(), dataspace->getId(), H5S_ALL, H5P_DEFAULT, value[0].data()); delete[] tmp; delete dataset; delete dataspace; write(full_name, dims, HDF5_TYPE<Scalar>::get_PredType(), value.data()); } template <typename domain_type> Loading Loading @@ -384,7 +369,7 @@ template <typename scalar_type> void HDF5Writer::execute(const std::string& name, const dca::linalg::Vector<std::complex<scalar_type>, dca::linalg::CPU>& V) { std::string full_name = get_path() + "/" + name; write(full_name, std::vector<hsize_t>{2, V.size()}, HDF5_TYPE<scalar_type>::get_PredType(), write(full_name, std::vector<hsize_t>{V.size(), 2}, HDF5_TYPE<scalar_type>::get_PredType(), V.ptr()); } Loading @@ -407,7 +392,7 @@ void HDF5Writer::execute(const std::string& name, template <typename scalar_type> void HDF5Writer::execute(const std::string& name, const dca::linalg::Matrix<std::complex<scalar_type>, dca::linalg::CPU>& A) { std::vector<hsize_t> dims{hsize_t(2), hsize_t(A.nrRows()), hsize_t(A.nrCols())}; std::vector<hsize_t> dims{hsize_t(A.nrRows()), hsize_t(A.nrCols()), hsize_t(2)}; std::vector<std::complex<scalar_type>> linearized(A.nrRows() * A.nrCols()); int linindex = 0; Loading src/io/hdf5/hdf5_reader.cpp +53 −41 Original line number Diff line number Diff line Loading @@ -6,6 +6,7 @@ // See CITATION.md for citation guidelines, if DCA++ is used for scientific publications. // // Author: Peter Staar (taa@zurich.ibm.com) // Giovanni Balduzzi (gbalduzz@itp.phys.ethz.ch) // // This file implements hdf5_reader.hpp. Loading @@ -19,7 +20,7 @@ namespace io { // dca::io:: HDF5Reader::~HDF5Reader() { if (my_file != NULL) if (file_) close_file(); } Loading @@ -35,56 +36,50 @@ void HDF5Reader::open_file(std::string file_name) { } } my_file = new H5::H5File(file_name.c_str(), H5F_ACC_RDONLY); file_ = std::make_unique<H5::H5File>(file_name.c_str(), H5F_ACC_RDONLY); } void HDF5Reader::close_file() { my_file->close(); delete my_file; my_file = NULL; file_->close(); file_.release(); } std::string HDF5Reader::get_path() { std::string path = "/"; for (size_t i = 0; i < my_paths.size(); i++) { path = path + my_paths[i]; for (size_t i = 0; i < paths_.size(); i++) { path = path + paths_[i]; if (i < my_paths.size() - 1) if (i < paths_.size() - 1) path = path + "/"; } return path; } bool HDF5Reader::execute(std::string name, std::string& value) //, H5File& file, std::string path) { bool HDF5Reader::execute(std::string name, std::string& value) { std::string full_name = get_path() + "/" + name; try { H5::DataSet dataset = my_file->openDataSet(full_name.c_str()); value.resize(dataset.getInMemDataSize(), 'a'); if (!exists(full_name)) { return false; } H5::DataSpace dataspace = dataset.getSpace(); auto dims = readSize(full_name); assert(dims.size() == 1); value.resize(dims.at(0)); H5Dread(dataset.getId(), HDF5_TYPE<char>::get(), dataspace.getId(), H5S_ALL, H5P_DEFAULT, &value[0]); read(full_name, HDF5_TYPE<char>::get_PredType(), value.data()); return true; } catch (const H5::FileIException& err) { std::cout << "\n\n\t the variable (" + name + ") does not exist in path : " + get_path() + "\n\n"; bool HDF5Reader::execute(std::string name, std::vector<std::string>& value) { std::string full_name = get_path() + "/" + name; if (!exists(full_name)) { return false; } } bool HDF5Reader::execute(std::string name, std::vector<std::string>& value) { open_group(name); bool success = true; try { int size = -1; execute("size", size); Loading @@ -94,20 +89,37 @@ bool HDF5Reader::execute(std::string name, std::vector<std::string>& value) { for (size_t l = 0; l < value.size(); l++) { open_group(std::to_string(l)); execute(std::to_string(l), value[l]); close_group(); } } catch (const H5::FileIException& err) { std::cout << "\n\n\t the variable (" + name + ") does not exist in path : " + get_path() + "\n\n"; success = false; } close_group(); close_group(); return success; return true; } void HDF5Reader::read(const std::string& name, H5::PredType type, void* data) const { H5::DataSet dataset = file_->openDataSet(name.c_str()); dataset.read(data, type); } bool HDF5Reader::exists(const std::string& name) const { auto code = H5Gget_objinfo(file_->getId(), name.c_str(), 0, NULL); return code == 0; } std::vector<hsize_t> HDF5Reader::readSize(const std::string& name) const { H5::DataSet dataset = file_->openDataSet(name.c_str()); H5::DataSpace dataspace = dataset.getSpace(); int n_dims = dataspace.getSimpleExtentNdims(); std::vector<hsize_t> dims(n_dims); dataspace.getSimpleExtentDims(dims.data(), nullptr); return dims; } } // namespace io Loading src/io/hdf5/hdf5_writer.cpp +1 −1 Original line number Diff line number Diff line Loading @@ -131,7 +131,7 @@ void HDF5Writer::write(const std::string& name, const std::vector<hsize_t>& dims } } bool HDF5Writer::exists(const std::string& name) { bool HDF5Writer::exists(const std::string& name) const { auto code = H5Gget_objinfo(file_id_, name.c_str(), 0, NULL); return code == 0; } Loading test/unit/io/hdf5_reader_writer_test.cpp +25 −0 Original line number Diff line number Diff line Loading @@ -95,6 +95,31 @@ TEST(HDF5ReaderWriterTest, VectorReadWrite) { reader.close_file(); } TEST(HDF5ReaderWriterTest, VectorOfVectorsReadWrite) { const std::string object_name = "a_vector"; const std::string file_name = "test_vector_vector.hdf5"; const std::vector<std::vector<double>> data_unequal_size{{0, 0, 2}, {1}, {1, 0}, {0, 0}}; const std::vector<std::vector<double>> data_equal_size{{0, 0}, {0, 1}, {1, 0}, {0, 0}}; for (const auto& input : {data_unequal_size, data_equal_size}) { // Create test file. dca::io::HDF5Writer writer; writer.open_file(file_name); writer.execute(object_name, input); writer.close_file(); // Read test file. dca::io::HDF5Reader reader; std::vector<std::vector<double>> data_read; reader.open_file(file_name); EXPECT_TRUE(reader.execute(object_name, data_read)); EXPECT_EQ(input, data_read); reader.close_file(); } } TEST(HDF5ReaderWriterTest, VectorOfStringsReadWrite) { std::vector<std::string> s1{"foo", "bar", "baz"}; Loading Loading
include/dca/io/hdf5/hdf5_reader.hpp +154 −178 Original line number Diff line number Diff line Loading @@ -34,7 +34,7 @@ public: typedef H5::H5File file_type; // In: verbose. If true, the reader outputs a short log whenever it is executed. HDF5Reader(bool verbose = true) : my_file(NULL), my_paths(0), verbose_(verbose) {} HDF5Reader(bool verbose = true) : verbose_(verbose) {} ~HDF5Reader(); Loading @@ -49,10 +49,10 @@ public: void close_file(); void open_group(std::string name) { my_paths.push_back(name); paths_.push_back(name); } void close_group() { my_paths.pop_back(); paths_.pop_back(); } std::string get_path(); Loading @@ -65,11 +65,14 @@ public: template <typename scalartype> bool execute(std::string name, scalartype& value); template <typename scalar_type> bool execute(std::string name, std::vector<scalar_type>& value); template <typename Scalar> bool execute(std::string name, std::vector<Scalar>& value); template <typename scalar_type> bool execute(std::string name, std::vector<std::complex<scalar_type>>& value); template <typename Scalar> bool execute(std::string name, std::vector<std::complex<Scalar>>& value); template <typename Scalar> bool execute(std::string name, std::vector<std::vector<Scalar>>& value); bool execute(std::string name, std::string& value); Loading @@ -87,30 +90,33 @@ public: template <typename scalartype, typename domain_type> bool execute(std::string name, func::function<scalartype, domain_type>& f); template <typename scalar_type> bool execute(std::string name, dca::linalg::Vector<scalar_type, dca::linalg::CPU>& A); template <typename Scalar> bool execute(std::string name, dca::linalg::Vector<Scalar, dca::linalg::CPU>& A); template <typename scalar_type> bool execute(std::string name, dca::linalg::Vector<std::complex<scalar_type>, dca::linalg::CPU>& A); template <typename Scalar> bool execute(std::string name, dca::linalg::Vector<std::complex<Scalar>, dca::linalg::CPU>& A); template <typename scalar_type> bool execute(std::string name, dca::linalg::Matrix<scalar_type, dca::linalg::CPU>& A); template <typename Scalar> bool execute(std::string name, dca::linalg::Matrix<Scalar, dca::linalg::CPU>& A); template <typename scalar_type> bool execute(std::string name, dca::linalg::Matrix<std::complex<scalar_type>, dca::linalg::CPU>& A); template <typename Scalar> bool execute(std::string name, dca::linalg::Matrix<std::complex<Scalar>, dca::linalg::CPU>& A); template <typename scalar_type> bool execute(dca::linalg::Matrix<scalar_type, dca::linalg::CPU>& A); template <typename Scalar> bool execute(dca::linalg::Matrix<Scalar, dca::linalg::CPU>& A); bool execute(std::string name, io::Buffer& buff) { return execute(name, static_cast<io::Buffer::Container&>(buff)); } private: bool fexists(const char* filename); bool exists(const std::string& name) const; void read(const std::string& name, H5::PredType type, void* data) const; std::vector<hsize_t> readSize(const std::string& name) const; H5::H5File* my_file; std::vector<std::string> my_paths; std::unique_ptr<H5::H5File> file_; std::vector<std::string> paths_; bool verbose_; }; Loading @@ -123,69 +129,90 @@ void HDF5Reader::from_file(arbitrary_struct_t& arbitrary_struct, std::string fil reader_obj.close_file(); } template <typename scalar_type> bool HDF5Reader::execute(std::string name, scalar_type& value) { template <typename Scalar> bool HDF5Reader::execute(std::string name, Scalar& value) { std::string full_name = get_path() + "/" + name; try { H5::DataSet dataset = my_file->openDataSet(full_name.c_str()); H5::DataSpace dataspace = dataset.getSpace(); H5Dread(dataset.getId(), HDF5_TYPE<scalar_type>::get(), dataspace.getId(), H5S_ALL, H5P_DEFAULT, &value); if (!exists(full_name)) { return false; } read(full_name, HDF5_TYPE<Scalar>::get_PredType(), &value); return true; } catch (...) { std::cout << "\n\n\t the variable (" + name + ") does not exist in path : " + get_path() + "\n\n"; template <typename Scalar> bool HDF5Reader::execute(std::string name, std::vector<Scalar>& value) { std::string full_name = get_path() + "/" + name; if (!exists(full_name)) { return false; } auto dims = readSize(full_name); assert(dims.size() == 1); value.resize(dims.at(0)); read(full_name, HDF5_TYPE<Scalar>::get_PredType(), value.data()); return true; } template <typename scalar_type> bool HDF5Reader::execute(std::string name, std::vector<scalar_type>& value) { template <typename Scalar> bool HDF5Reader::execute(std::string name, std::vector<std::complex<Scalar>>& value) { std::string full_name = get_path() + "/" + name; try { H5::DataSet dataset = my_file->openDataSet(full_name.c_str()); value.resize(dataset.getInMemDataSize() / sizeof(scalar_type)); if (!exists(full_name)) { return false; } H5::DataSpace dataspace = dataset.getSpace(); auto dims = readSize(full_name); assert(dims.size() == 2); value.resize(dims.at(0)); H5Dread(dataset.getId(), HDF5_TYPE<scalar_type>::get(), dataspace.getId(), H5S_ALL, H5P_DEFAULT, &value[0]); read(full_name, HDF5_TYPE<Scalar>::get_PredType(), value.data()); return true; } catch (...) { std::cout << "\n\n\t the variable (" + name + ") does not exist in path : " + get_path() + "\n\n"; template <typename Scalar> bool HDF5Reader::execute(std::string name, std::vector<std::vector<Scalar>>& value) { std::string full_name = get_path() + "/" + name; if (!exists(full_name)) { return false; } } template <typename scalar_type> bool HDF5Reader::execute(std::string name, std::vector<std::complex<scalar_type>>& value) { std::string full_name = get_path() + "/" + name; const bool equal_size = !exists(full_name + "/data"); try { H5::DataSet dataset = my_file->openDataSet(full_name.c_str()); if (equal_size) { auto dims = readSize(full_name); assert(dims.size() == 2); std::vector<Scalar> linearized(dims[0] * dims[1]); value.resize(dataset.getInMemDataSize() / sizeof(std::complex<scalar_type>)); read(full_name, HDF5_TYPE<Scalar>::get_PredType(), linearized.data()); value.resize(dims[0]); const Scalar* read_location = linearized.data(); for (auto& v : value) { v.resize(dims[1]); std::copy_n(read_location, dims[1], v.data()); read_location += dims[1]; } } else { open_group(name); H5::DataSpace dataspace = dataset.getSpace(); int size = -1; execute("size", size); value.resize(size); H5Dread(dataset.getId(), HDF5_TYPE<scalar_type>::get(), dataspace.getId(), H5S_ALL, H5P_DEFAULT, &value[0]); return true; open_group("data"); for (int i = 0; i < value.size(); ++i) { execute("row_" + std::to_string(i), value[i]); } catch (...) { std::cout << "\n\n\t the variable (" + name + ") does not exist in path : " + get_path() + "\n\n"; return false; close_group(); close_group(); } return true; } template <typename scalartype, typename domain_type> Loading @@ -195,157 +222,106 @@ bool HDF5Reader::execute(func::function<scalartype, domain_type>& f) { template <typename scalartype, typename domain_type> bool HDF5Reader::execute(std::string name, func::function<scalartype, domain_type>& f) { std::cout << "\n\tstart reading function : " << name; open_group(name); bool success = true; try { std::string full_name = get_path() + "/data"; H5::DataSet dataset = my_file->openDataSet(full_name.c_str()); H5::DataSpace dataspace = dataset.getSpace(); std::string full_name = get_path() + "/" + name; H5Dread(dataset.getId(), HDF5_TYPE<scalartype>::get(), dataspace.getId(), H5S_ALL, H5P_DEFAULT, &f(0)); } catch (const H5::FileIException& err) { if (!exists(full_name)) { std::cout << "\n\n\t the function (" + name + ") does not exist in path : " + get_path() + "\n\n"; success = false; } close_group(); return success; return false; } template <typename scalar_type> bool HDF5Reader::execute(std::string name, dca::linalg::Vector<scalar_type, dca::linalg::CPU>& V) { bool success = true; try { std::string full_name = get_path() + "/" + name; std::cout << "\n\tstart reading function : " << name; H5::DataSet dataset = my_file->openDataSet(full_name.c_str()); read(full_name + "/data", HDF5_TYPE<scalartype>::get_PredType(), f.values()); V.resize(dataset.getInMemDataSize() / sizeof(scalar_type)); return true; } H5::DataSpace dataspace = dataset.getSpace(); template <typename Scalar> bool HDF5Reader::execute(std::string name, dca::linalg::Vector<Scalar, dca::linalg::CPU>& V) { std::string full_name = get_path() + "/" + name; if (!exists(full_name)) { return false; } H5Dread(dataset.getId(), HDF5_TYPE<scalar_type>::get(), dataspace.getId(), H5S_ALL, H5P_DEFAULT, &V[0]); auto dims = readSize(full_name); assert(dims.size() == 1); V.resize(dims.at(0)); V.set_name(name); } catch (const H5::FileIException& err) { std::cout << "\n\n\t the vector (" + name + ") does not exist in path : " + get_path() + "\n\n"; success = false; } read(full_name, HDF5_TYPE<Scalar>::get_PredType(), V.ptr()); return success; return true; } template <typename scalar_type> template <typename Scalar> bool HDF5Reader::execute(std::string name, dca::linalg::Vector<std::complex<scalar_type>, dca::linalg::CPU>& V) { bool success = true; try { dca::linalg::Vector<std::complex<Scalar>, dca::linalg::CPU>& V) { std::string full_name = get_path() + "/" + name; if (!exists(full_name)) { return false; } H5::DataSet dataset = my_file->openDataSet(full_name.c_str()); V.resize(dataset.getInMemDataSize() / sizeof(std::complex<scalar_type>)); H5::DataSpace dataspace = dataset.getSpace(); H5Dread(dataset.getId(), HDF5_TYPE<scalar_type>::get(), dataspace.getId(), H5S_ALL, H5P_DEFAULT, &V[0]); auto dims = readSize(full_name); assert(dims.size() == 2); V.resize(dims.at(0)); V.set_name(name); } catch (const H5::FileIException& err) { std::cout << "\n\n\t the vector (" + name + ") does not exist in path : " + get_path() + "\n\n"; success = false; } read(full_name, HDF5_TYPE<Scalar>::get_PredType(), V.ptr()); return success; return true; } template <typename scalar_type> bool HDF5Reader::execute(std::string name, dca::linalg::Matrix<scalar_type, dca::linalg::CPU>& A) { bool success = true; try { template <typename Scalar> bool HDF5Reader::execute(std::string name, dca::linalg::Matrix<Scalar, dca::linalg::CPU>& A) { std::string full_name = get_path() + "/" + name; if (!exists(full_name)) { return false; } H5::DataSet dataset = my_file->openDataSet(full_name.c_str()); H5::DataSpace dataspace = dataset.getSpace(); std::array<hsize_t, 2> dims; dataspace.getSimpleExtentDims(dims.data(), nullptr); auto dims = readSize(full_name); assert(dims.size() == 2); std::vector<scalar_type> linearized(dims[0] * dims[1]); dataset.read(linearized.data(), HDF5_TYPE<scalar_type>::get_PredType(), dataspace); std::vector<Scalar> linearized(dims[0] * dims[1]); read(full_name, HDF5_TYPE<Scalar>::get_PredType(), linearized.data()); // HDF5 is column major, while Matrix is not major. // HDF5 is column major, while Matrix is row major. A.resizeNoCopy(std::make_pair(dims[0], dims[1])); unsigned linindex = 0; for (int i = 0; i < A.nrRows(); ++i) { for (int i = 0, linindex = 0; i < A.nrRows(); ++i) { for (int j = 0; j < A.nrCols(); ++j) A(i, j) = linearized[linindex++]; } A.set_name(name); } catch (const H5::FileIException& err) { std::cout << "\n\n\t the function (" + name + ") does not exist in path : " + get_path() + "\n\n"; success = false; } return success; return true; } template <typename scalar_type> template <typename Scalar> bool HDF5Reader::execute(std::string name, dca::linalg::Matrix<std::complex<scalar_type>, dca::linalg::CPU>& A) { bool success = true; try { dca::linalg::Matrix<std::complex<Scalar>, dca::linalg::CPU>& A) { std::string full_name = get_path() + "/" + name; if (!exists(full_name)) { return false; } H5::DataSet dataset = my_file->openDataSet(full_name.c_str()); H5::DataSpace dataspace = dataset.getSpace(); std::array<hsize_t, 3> dims; dataspace.getSimpleExtentDims(dims.data(), nullptr); A.resizeNoCopy(std::make_pair(dims[1], dims[2])); std::vector<std::complex<scalar_type>> linearized(A.nrRows() * A.nrCols()); auto dims = readSize(full_name); assert(dims.size() == 3); dataset.read(linearized.data(), HDF5_TYPE<scalar_type>::get_PredType(), dataspace); std::vector<std::complex<Scalar>> linearized(dims[0] * dims[1]); read(full_name, HDF5_TYPE<Scalar>::get_PredType(), linearized.data()); // HDF5 is column major, while Matrix is not major. unsigned linindex = 0; for (int i = 0; i < A.nrRows(); ++i) for (int j = 0; j < A.nrCols(); ++j) { // HDF5 is column major, while Matrix is row major. A.resizeNoCopy(std::make_pair(dims[0], dims[1])); for (int i = 0, linindex = 0; i < A.nrRows(); ++i) { for (int j = 0; j < A.nrCols(); ++j) A(i, j) = linearized[linindex++]; } A.set_name(name); } catch (const H5::FileIException& err) { std::cout << "\n\n\t the function (" + name + ") does not exist in path : " + get_path() + "\n\n"; success = false; } return success; return true; } template <typename scalar_type> bool HDF5Reader::execute(dca::linalg::Matrix<scalar_type, dca::linalg::CPU>& A) { template <typename Scalar> bool HDF5Reader::execute(dca::linalg::Matrix<Scalar, dca::linalg::CPU>& A) { return execute(A.get_name(), A); } Loading
include/dca/io/hdf5/hdf5_writer.hpp +17 −32 Original line number Diff line number Diff line Loading @@ -145,7 +145,7 @@ public: private: bool fexists(const char* filename); bool exists(const std::string& name); bool exists(const std::string& name) const; void write(const std::string& name, const std::vector<hsize_t>& size, H5::PredType type, const void* data); Loading Loading @@ -203,7 +203,7 @@ void HDF5Writer::execute(const std::string& name, const std::vector<std::complex<scalar_type>>& value) { if (value.size() > 0) { std::string full_name = get_path() + "/" + name; std::vector<hsize_t> dims{2, value.size()}; std::vector<hsize_t> dims{value.size(), 2}; write(full_name, dims, HDF5_TYPE<scalar_type>::get_PredType(), value.data()); } } Loading @@ -226,21 +226,27 @@ void HDF5Writer::execute(const std::string& name, const std::vector<std::vector< std::vector<hsize_t> dims{value.size(), cols}; std::vector<scalar_type> linearized(dims[0] * dims[1]); std::size_t linindex = 0; for (std::size_t i = 0; i < value.size(); ++i) for (std::size_t j = 0; j < cols; ++j, ++linindex) linearized[linindex] = value[i][j]; for (std::size_t i = 0, linindex = 0; i < value.size(); ++i) for (std::size_t j = 0; j < cols; ++j) linearized[linindex++] = value[i][j]; write(full_name, dims, HDF5_TYPE<scalar_type>::get_PredType(), value.data()); write(full_name, dims, HDF5_TYPE<scalar_type>::get_PredType(), linearized.data()); } else { open_group(full_name); execute("size", value.size()); open_group("data"); std::vector<hsize_t> dims(1); for (std::size_t i = 0; i < value.size(); ++i) { const std::string new_name = full_name + "/row_" + std::to_string(i); const std::string new_name = full_name + "/data/row_" + std::to_string(i); dims[0] = value[i].size(); write(new_name, dims, HDF5_TYPE<scalar_type>::get_PredType(), value[i].data()); } close_group(); close_group(); } } Loading @@ -252,30 +258,9 @@ void HDF5Writer::execute(const std::string& name, const std::vector<std::array<S return; std::array<hsize_t, 2> dims{value.size(), n}; H5::DataSet* dataset = nullptr; H5::DataSpace* dataspace = nullptr; std::string full_name = get_path() + "/" + name; dataspace = new H5::DataSpace(2, dims.data()); dataset = new H5::DataSet( file_->createDataSet(full_name.c_str(), HDF5_TYPE<Scalar>::get_PredType(), *dataspace)); Scalar* tmp = new Scalar[dims[0] * dims[1]]; // hdf5 has row-major ordering! // for (hsize_t i = 0; i < dims[0]; i++) // for (hsize_t j = 0; j < dims[1]; j++) // tmp[i * dims[1] + j] = value[i][j]; H5Dwrite(dataset->getId(), HDF5_TYPE<Scalar>::get(), dataspace->getId(), H5S_ALL, H5P_DEFAULT, value[0].data()); delete[] tmp; delete dataset; delete dataspace; write(full_name, dims, HDF5_TYPE<Scalar>::get_PredType(), value.data()); } template <typename domain_type> Loading Loading @@ -384,7 +369,7 @@ template <typename scalar_type> void HDF5Writer::execute(const std::string& name, const dca::linalg::Vector<std::complex<scalar_type>, dca::linalg::CPU>& V) { std::string full_name = get_path() + "/" + name; write(full_name, std::vector<hsize_t>{2, V.size()}, HDF5_TYPE<scalar_type>::get_PredType(), write(full_name, std::vector<hsize_t>{V.size(), 2}, HDF5_TYPE<scalar_type>::get_PredType(), V.ptr()); } Loading @@ -407,7 +392,7 @@ void HDF5Writer::execute(const std::string& name, template <typename scalar_type> void HDF5Writer::execute(const std::string& name, const dca::linalg::Matrix<std::complex<scalar_type>, dca::linalg::CPU>& A) { std::vector<hsize_t> dims{hsize_t(2), hsize_t(A.nrRows()), hsize_t(A.nrCols())}; std::vector<hsize_t> dims{hsize_t(A.nrRows()), hsize_t(A.nrCols()), hsize_t(2)}; std::vector<std::complex<scalar_type>> linearized(A.nrRows() * A.nrCols()); int linindex = 0; Loading
src/io/hdf5/hdf5_reader.cpp +53 −41 Original line number Diff line number Diff line Loading @@ -6,6 +6,7 @@ // See CITATION.md for citation guidelines, if DCA++ is used for scientific publications. // // Author: Peter Staar (taa@zurich.ibm.com) // Giovanni Balduzzi (gbalduzz@itp.phys.ethz.ch) // // This file implements hdf5_reader.hpp. Loading @@ -19,7 +20,7 @@ namespace io { // dca::io:: HDF5Reader::~HDF5Reader() { if (my_file != NULL) if (file_) close_file(); } Loading @@ -35,56 +36,50 @@ void HDF5Reader::open_file(std::string file_name) { } } my_file = new H5::H5File(file_name.c_str(), H5F_ACC_RDONLY); file_ = std::make_unique<H5::H5File>(file_name.c_str(), H5F_ACC_RDONLY); } void HDF5Reader::close_file() { my_file->close(); delete my_file; my_file = NULL; file_->close(); file_.release(); } std::string HDF5Reader::get_path() { std::string path = "/"; for (size_t i = 0; i < my_paths.size(); i++) { path = path + my_paths[i]; for (size_t i = 0; i < paths_.size(); i++) { path = path + paths_[i]; if (i < my_paths.size() - 1) if (i < paths_.size() - 1) path = path + "/"; } return path; } bool HDF5Reader::execute(std::string name, std::string& value) //, H5File& file, std::string path) { bool HDF5Reader::execute(std::string name, std::string& value) { std::string full_name = get_path() + "/" + name; try { H5::DataSet dataset = my_file->openDataSet(full_name.c_str()); value.resize(dataset.getInMemDataSize(), 'a'); if (!exists(full_name)) { return false; } H5::DataSpace dataspace = dataset.getSpace(); auto dims = readSize(full_name); assert(dims.size() == 1); value.resize(dims.at(0)); H5Dread(dataset.getId(), HDF5_TYPE<char>::get(), dataspace.getId(), H5S_ALL, H5P_DEFAULT, &value[0]); read(full_name, HDF5_TYPE<char>::get_PredType(), value.data()); return true; } catch (const H5::FileIException& err) { std::cout << "\n\n\t the variable (" + name + ") does not exist in path : " + get_path() + "\n\n"; bool HDF5Reader::execute(std::string name, std::vector<std::string>& value) { std::string full_name = get_path() + "/" + name; if (!exists(full_name)) { return false; } } bool HDF5Reader::execute(std::string name, std::vector<std::string>& value) { open_group(name); bool success = true; try { int size = -1; execute("size", size); Loading @@ -94,20 +89,37 @@ bool HDF5Reader::execute(std::string name, std::vector<std::string>& value) { for (size_t l = 0; l < value.size(); l++) { open_group(std::to_string(l)); execute(std::to_string(l), value[l]); close_group(); } } catch (const H5::FileIException& err) { std::cout << "\n\n\t the variable (" + name + ") does not exist in path : " + get_path() + "\n\n"; success = false; } close_group(); close_group(); return success; return true; } void HDF5Reader::read(const std::string& name, H5::PredType type, void* data) const { H5::DataSet dataset = file_->openDataSet(name.c_str()); dataset.read(data, type); } bool HDF5Reader::exists(const std::string& name) const { auto code = H5Gget_objinfo(file_->getId(), name.c_str(), 0, NULL); return code == 0; } std::vector<hsize_t> HDF5Reader::readSize(const std::string& name) const { H5::DataSet dataset = file_->openDataSet(name.c_str()); H5::DataSpace dataspace = dataset.getSpace(); int n_dims = dataspace.getSimpleExtentNdims(); std::vector<hsize_t> dims(n_dims); dataspace.getSimpleExtentDims(dims.data(), nullptr); return dims; } } // namespace io Loading
src/io/hdf5/hdf5_writer.cpp +1 −1 Original line number Diff line number Diff line Loading @@ -131,7 +131,7 @@ void HDF5Writer::write(const std::string& name, const std::vector<hsize_t>& dims } } bool HDF5Writer::exists(const std::string& name) { bool HDF5Writer::exists(const std::string& name) const { auto code = H5Gget_objinfo(file_id_, name.c_str(), 0, NULL); return code == 0; } Loading
test/unit/io/hdf5_reader_writer_test.cpp +25 −0 Original line number Diff line number Diff line Loading @@ -95,6 +95,31 @@ TEST(HDF5ReaderWriterTest, VectorReadWrite) { reader.close_file(); } TEST(HDF5ReaderWriterTest, VectorOfVectorsReadWrite) { const std::string object_name = "a_vector"; const std::string file_name = "test_vector_vector.hdf5"; const std::vector<std::vector<double>> data_unequal_size{{0, 0, 2}, {1}, {1, 0}, {0, 0}}; const std::vector<std::vector<double>> data_equal_size{{0, 0}, {0, 1}, {1, 0}, {0, 0}}; for (const auto& input : {data_unequal_size, data_equal_size}) { // Create test file. dca::io::HDF5Writer writer; writer.open_file(file_name); writer.execute(object_name, input); writer.close_file(); // Read test file. dca::io::HDF5Reader reader; std::vector<std::vector<double>> data_read; reader.open_file(file_name); EXPECT_TRUE(reader.execute(object_name, data_read)); EXPECT_EQ(input, data_read); reader.close_file(); } } TEST(HDF5ReaderWriterTest, VectorOfStringsReadWrite) { std::vector<std::string> s1{"foo", "bar", "baz"}; Loading