Add abilty to open data sets in HDF5. (ac86612c) · Commits · Cianciosa, Mark / ml_model_embedder

.gitignore

+3 −0

Original line number	Diff line number	Diff line
		build
		.DS_Store
		LIBZIP
		MLX

source/hdf5.hpp

0 → 100644

+187 −0

Original line number	Diff line number	Diff line
		//------------------------------------------------------------------------------
		/// @file hdf5.hpp
		/// @brief Base class for the hdf5 files.
		//------------------------------------------------------------------------------

		#ifndef hdf5_hpp
		#define hdf5_hpp

		#include <string>
		#include <cassert>
		#include <iostream>

		#include <hdf5.h>
		#include <hdf5_hl.h>

		namespace ml_embeder {
		namespace hdf5 {
		//------------------------------------------------------------------------------
		/// @brief A hdf5 file.
		//------------------------------------------------------------------------------
		class file {
		private:
		/// hdf5 file reference.
		const hid_t f;

		//------------------------------------------------------------------------------
		/// @brief Check errors in hdf5 functions.
		///
		/// @param[in] status The error status.
		//------------------------------------------------------------------------------
		static void check_error(const herr_t status) {
		assert(status == 0 && "hdf5 file error");
		#ifdef NDEBUG
		if (status) {
		std::cerr << "hdf5 file error" << std::endl;
		exit(status);
		}
		#endif
		}

		public:
		//------------------------------------------------------------------------------
		/// @brief A hdf5 group.
		//------------------------------------------------------------------------------
		class group {
		private:
		/// hdf5 group reference.
		const hid_t g;

		public:
		//------------------------------------------------------------------------------
		/// @brief A hdf5 dataset.
		//------------------------------------------------------------------------------
		class dataset {
		private:
		/// The hdf5 dataset reference.
		const hid_t d;

		public:
		//------------------------------------------------------------------------------
		/// @brief Open a group.
		///
		/// @param[in] loc_id The location id.
		/// @param[in] datasetname Name of the dataset.
		//------------------------------------------------------------------------------
		dataset(const hid_t loc_id,
		const std::string datasetname) :
		d(H5Dopen(loc_id, datasetname.c_str(), 0)) {
		assert(d != H5I_INVALID_HID &&
		"Failed to open dataset.");
		#ifdef NDEBUG
		if (d == H5I_INVALID_HID) {
		std::cerr << "Failed to open dataset: "
		<< datasetname << std::endl;
		exit(d);
		}
		#endif
		}

		//------------------------------------------------------------------------------
		/// @brief Destruct a hdf5 group.
		//------------------------------------------------------------------------------
		~dataset() {
		check_error(H5Dclose(d));
		}
		};

		//------------------------------------------------------------------------------
		/// @brief Open a group.
		///
		/// @param[in] loc_id The location id.
		/// @param[in] groupname Name of the group.
		//------------------------------------------------------------------------------
		group(const hid_t loc_id,
		const std::string groupname) :
		g(H5Gopen(loc_id, groupname.c_str(), 0)) {
		assert(g != H5I_INVALID_HID && "Failed to open group.");
		#ifdef NDEBUG
		if (g == H5I_INVALID_HID) {
		std::cerr << "Failed to open group: " << groupname
		<< std::endl;
		exit(g);
		}
		#endif
		}

		//------------------------------------------------------------------------------
		/// @brief Destruct a hdf5 group.
		//------------------------------------------------------------------------------
		~group() {
		check_error(H5Gclose(g));
		}

		//------------------------------------------------------------------------------
		/// @brief Open group.
		///
		/// @param[in] groupname Name of the group.
		/// @return A @ref ml_embeder::hdf5_file::hdf5_group.
		//------------------------------------------------------------------------------
		group open_group(const std::string groupname) {
		return group(g, groupname);
		}

		//------------------------------------------------------------------------------
		/// @brief Open group.
		///
		/// @param[in] groupname Name of the group.
		/// @return A @ref ml_embeder::hdf5_file::hdf5_group.
		//------------------------------------------------------------------------------
		dataset open_dataset(const std::string datasetname) {
		return dataset(g, datasetname);
		}
		};

		//------------------------------------------------------------------------------
		/// @brief Initalize the hdf5 library.
		//------------------------------------------------------------------------------
		static void initalize() {
		check_error(H5open());
		}

		//------------------------------------------------------------------------------
		/// @brief Initalize the hdf5 library.
		//------------------------------------------------------------------------------
		static void finalize() {
		check_error(H5close());
		}

		//------------------------------------------------------------------------------
		/// @brief Open a hdf5 file.
		///
		/// @param[in] buffer The file data buffer.
		/// @param[in] size The buffer size.
		//------------------------------------------------------------------------------
		file(void *buffer, const size_t size) :
		f(H5LTopen_file_image(buffer, size,
		H5LT_FILE_IMAGE_DONT_COPY \|
		H5LT_FILE_IMAGE_DONT_RELEASE)) {
		assert(f != H5I_INVALID_HID && "Failed to open file image.");
		#ifdef NDEBUG
		if (f == H5I_INVALID_HID) {
		std::cerr << "Failed to open image" << std::endl;
		exit(f);
		}
		#endif
		}
		//------------------------------------------------------------------------------
		/// @brief Destruct a hdf5 file.
		//------------------------------------------------------------------------------
		~file() {
		check_error(H5Fclose(f));
		}

		//------------------------------------------------------------------------------
		/// @brief Open group.
		///
		/// @param[in] groupname Name of the group.
		/// @return A @ref ml_embeder::hdf5_file::hdf5_group.
		//------------------------------------------------------------------------------
		group open_group(const std::string groupname) {
		return group(f, groupname);
		}
		};
		}
		}

		#endif /* hdf5_hpp */

source/ml_embeder.hpp

0 → 100644

+2 −0

Original line number	Diff line number	Diff line
		#include <hdf5.hpp>
		#include <zip.hpp>

source/zip.hpp

+6 −7

Original line number	Diff line number	Diff line
		@@ -14,7 +14,7 @@ namespace ml_embeder {
		//------------------------------------------------------------------------------
		/// @brief A zip file.
		//------------------------------------------------------------------------------
		class zip_file {
		class zip {
		private:
		/// Zip archive.
		zip_t *z;
		@@ -76,7 +76,7 @@ namespace ml_embeder {
		///
		/// @param[in] filename The zip file.
		//------------------------------------------------------------------------------
		zip_file(const std::string filename) {
		zip(const std::string filename) {
		int err = 0;
		z = zip_open(filename.c_str(), ZIP_RDONLY, &err);
		if (!z \|\| err) {
		@@ -94,7 +94,7 @@ namespace ml_embeder {
		//------------------------------------------------------------------------------
		/// @brief Destruct a zip file.
		//------------------------------------------------------------------------------
		~zip_file() {
		~zip() {
		zip_close(z);
		}

		@@ -142,15 +142,14 @@ namespace ml_embeder {
		/// @param[in] filename The name of the file in the archieve.
		/// @returns A @ref ml_embeder::zip_file::file reference.
		//------------------------------------------------------------------------------
		zip_file::file get_file(const std::string filename) {
		zip::file get_file(const std::string filename) {
		zip_int64_t index = get_file_index(filename);
		check_error();

		zip_stat_t stat;
		zip_stat_index(z, index, ZIP_STAT_SIZE, &stat);
		zip_stat_index(z, index, ZIP_STAT_SIZE \| ZIP_STAT_COMP_SIZE, &stat);

		zip_file::file f(zip_fopen_index(z, index, ZIP_FL_COMPRESSED),
		stat.size);
		zip::file f(zip_fopen_index(z, index, ZIP_FL_UNCHANGED), stat.size);
		check_error();
		return f;
		}

tests/ml_embeder_test.cpp

+11 −4

Original line number	Diff line number	Diff line
		@@ -12,8 +12,15 @@
		/// @param[in] argv Array of commandline arguments.
		//------------------------------------------------------------------------------
		int main(int argc, const char * argv[]) {
		ml_embeder::zip_file zip("/Users/m4c/OneDrive - Oak Ridge National Laboratory/eped/eped_model7.1/saved_model.keras");
		ml_embeder::zip_file::file::buffer b = zip.get_file("model.weights.h5").get_buffer();

		ml_embeder::hdf5_file hf(b.data(), b.size());
		ml_embeder::hdf5::file::initalize(); {
		ml_embeder::zip zip("/Users/m4c/OneDrive - Oak Ridge National Laboratory/eped/eped_model7.1/saved_model.keras");
		ml_embeder::zip::file::buffer b = zip.get_file("model.weights.h5").get_buffer();
		ml_embeder::hdf5::file hf(b.data(), b.size());
		ml_embeder::hdf5::file::group hg1 = hf.open_group("layers");
		ml_embeder::hdf5::file::group hg2 = hg1.open_group("dense");
		ml_embeder::hdf5::file::group hg3 = hg2.open_group("vars");
		ml_embeder::hdf5::file::group::dataset hd1 = hg3.open_dataset("0");
		ml_embeder::hdf5::file::group::dataset hd2 = hg3.open_dataset("1");
		}
		ml_embeder::hdf5::file::finalize();
		}