Use variable length datasets. (19620539) · Commits · NDIP / Tool Sources / Direct-Geometry Spectroscopy / DCA / DCA Main

include/dca/io/hdf5/hdf5_reader.hpp

+12 −28

Original line number	Diff line number	Diff line
		@@ -117,7 +117,7 @@ public:
		private:
		bool exists(const std::string& name) const;

		void read(const std::string& name, H5::PredType type, void* data) const;
		void read(const std::string& name, H5::DataType type, void* data) const;
		std::vector<hsize_t> readSize(const std::string& name) const;

		std::unique_ptr<H5::H5File> file_;
		@@ -185,37 +185,21 @@ bool HDF5Reader::execute(const std::string& name, std::vector<std::vector<Scalar
		return false;
		}

		const bool equal_size = !exists(full_name + "/data");
		auto size = readSize(full_name)[0];
		const auto type = H5::VarLenType(HDF5_TYPE<Scalar>::get_PredType());

		if (equal_size) {
		auto dims = readSize(full_name);
		assert(dims.size() == 2);
		std::vector<Scalar> linearized(dims[0] * dims[1]);
		std::vector<hvl_t> data(size);

		read(full_name, HDF5_TYPE<Scalar>::get_PredType(), linearized.data());
		value.resize(dims[0]);
		const Scalar* read_location = linearized.data();
		for (auto& v : value) {
		v.resize(dims[1]);
		std::copy_n(read_location, dims[1], v.data());
		read_location += dims[1];
		}
		}
		else {
		open_group(name);
		H5::DataSet dataset = file_->openDataSet(name.c_str());
		dataset.read(data.data(), type);

		int size = -1;
		execute("size", size);
		value.resize(size);

		open_group("data");
		for (int i = 0; i < value.size(); ++i) {
		execute("row_" + std::to_string(i), value[i]);
		for (int i = 0; i < size; ++i) {
		value[i].resize(data[i].len);
		std::copy_n(static_cast<Scalar*>(data[i].p), data[i].len, value[i].data());
		}
		close_group();

		close_group();
		}
		dataset.vlenReclaim(data.data(), type, dataset.getSpace());

		return true;
		}

include/dca/io/hdf5/hdf5_writer.hpp

+9 −38

Original line number	Diff line number	Diff line
		@@ -211,46 +211,17 @@ void HDF5Writer::execute(const std::string& name, const std::vector<std::complex

		template <typename Scalar>
		void HDF5Writer::execute(const std::string& name, const std::vector<std::vector<Scalar>>& value) {
		if (value.size() > 0) {
		std::string full_name = get_path() + "/" + name;

		bool all_the_same_size = true;
		const std::size_t cols = value[0].size();
		for (auto& v : value) {
		if (v.size() != cols) {
		all_the_same_size = false;
		break;
		}
		}

		if (all_the_same_size) {
		std::vector<hsize_t> dims{value.size(), cols};
		std::vector<Scalar> linearized(dims[0] * dims[1]);

		for (std::size_t i = 0, linindex = 0; i < value.size(); ++i)
		for (std::size_t j = 0; j < cols; ++j)
		linearized[linindex++] = value[i][j];

		write(full_name, dims, HDF5_TYPE<Scalar>::get_PredType(), linearized.data());
		std::vector<hvl_t> data(value.size());
		for (int i = 0; i < value.size(); ++i) {
		data[i].p = const_cast<void>(static_cast<const void>((value[i].data())));
		data[i].len = value[i].size();
		}
		else {
		open_group(full_name);

		execute("size", value.size());
		const auto type = H5::VarLenType(HDF5_TYPE<Scalar>::get_PredType());

		open_group("data");

		std::vector<hsize_t> dims(1);
		for (std::size_t i = 0; i < value.size(); ++i) {
		const std::string new_name = full_name + "/data/row_" + std::to_string(i);
		dims[0] = value[i].size();
		write(new_name, dims, HDF5_TYPE<Scalar>::get_PredType(), value[i].data());
		}

		close_group();
		close_group();
		}
		}
		write(full_name, std::vector<hsize_t>{data.size()}, type, data.data());
		}

		template <typename Scalar, std::size_t n>

src/io/hdf5/hdf5_reader.cpp

+10 −11

Original line number	Diff line number	Diff line
		@@ -84,26 +84,25 @@ bool HDF5Reader::execute(const std::string& name, std::vector<std::string>& valu
		return false;
		}

		open_group(name);
		H5::DataSet dataset = file_->openDataSet(name.c_str());
		auto size = readSize(full_name)[0];
		auto s_type = H5::StrType(H5::PredType::C_S1, H5T_VARIABLE);

		int size = -1;
		execute("size", size);
		std::vector<char*> data(size);
		dataset.read(data.data(), s_type);

		value.resize(size);

		open_group("data");

		for (size_t l = 0; l < value.size(); l++) {
		execute(std::to_string(l), value[l]);
		for (int i = 0; i < size; ++i) {
		value[i] = data[i];
		}

		close_group();
		close_group();
		// clean memory
		dataset.vlenReclaim(data.data(), s_type, dataset.getSpace());

		return true;
		}

		void HDF5Reader::read(const std::string& name, H5::PredType type, void* data) const {
		void HDF5Reader::read(const std::string& name, H5::DataType type, void* data) const {
		H5::DataSet dataset = file_->openDataSet(name.c_str());
		dataset.read(data, type);
		}

src/io/hdf5/hdf5_writer.cpp

+9 −9

Original line number	Diff line number	Diff line
		@@ -98,19 +98,19 @@ void HDF5Writer::execute(const std::string& name,
		void HDF5Writer::execute(const std::string& name,
		const std::vector<std::string>& value) //, H5File& file, std::string path)
		{
		if (value.size() > 0) {
		open_group(name);
		execute("size", static_cast<int>(value.size()));
		if (value.size() == 0)
		return;

		open_group("data");
		const std::string full_name = get_path() + "/" + name;

		for (int i = 0; i < value.size(); ++i) {
		execute(std::to_string(i), value[i]);
		}
		auto s_type = H5::StrType(H5::PredType::C_S1, H5T_VARIABLE);

		close_group();
		close_group();
		std::vector<const char*> data;
		for (const auto& s : value) {
		data.push_back(s.data());
		}

		write(full_name, std::vector<hsize_t>{data.size()}, s_type, data.data());
		}

		H5::DataSet HDF5Writer::write(const std::string& name, const std::vector<hsize_t>& dims,

test/unit/io/hdf5_reader_writer_test.cpp

+13 −17

Original line number	Diff line number	Diff line
		@@ -98,14 +98,12 @@ TEST(HDF5ReaderWriterTest, VectorReadWrite) {
		TEST(HDF5ReaderWriterTest, VectorOfVectorsReadWrite) {
		const std::string object_name = "a_vector";
		const std::string file_name = "test_vector_vector.hdf5";
		const std::vector<std::vector<double>> data_unequal_size{{0, 0, 2}, {1}, {1, 0}, {0, 0}};
		const std::vector<std::vector<double>> data_equal_size{{0, 0}, {0, 1}, {1, 0}, {0, 0}};
		const std::vector<std::vector<double>> data_unequal_size{{0, 0, 2}, {1}, {1, 0}, {}, {0, 0}};

		for (const auto& input : {data_unequal_size, data_equal_size}) {
		// Create test file.
		dca::io::HDF5Writer writer;
		writer.open_file(file_name);
		writer.execute(object_name, input);
		writer.execute(object_name, data_unequal_size);
		writer.close_file();

		// Read test file.
		@@ -114,11 +112,9 @@ TEST(HDF5ReaderWriterTest, VectorOfVectorsReadWrite) {
		reader.open_file(file_name);
		EXPECT_TRUE(reader.execute(object_name, data_read));

		EXPECT_EQ(input, data_read);

		EXPECT_EQ(data_unequal_size, data_read);
		reader.close_file();
		}
		}

		TEST(HDF5ReaderWriterTest, VectorOfArraysReadWrite) {
		const std::string object_name = "obj";