Merge pull request #4059 from vicentebolea/backports-from-master (1f50ea27) · Commits · ecpcitest / ADIOS2

.github/workflows/everything.yml

+7 −1

Original line number	Diff line number	Diff line
		@@ -339,7 +339,7 @@ jobs:

		runs-on: ${{ matrix.image }}
		env:
		GH_YML_JOBNAME: ${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.parallel }}
		GH_YML_JOBNAME: ${{ matrix.os }}-${{ matrix.compiler }}${{ matrix.shared == 'static' && '-static' \|\| ''}}-${{ matrix.parallel }}
		GH_YML_BASE_OS: Windows
		GH_YML_MATRIX_OS: ${{ matrix.os }}
		GH_YML_MATRIX_COMPILER: ${{ matrix.compiler }}
		@@ -349,6 +349,7 @@ jobs:
		fail-fast: false
		matrix:
		os: [win2019, win2022]
		shared: [shared]
		parallel: [serial, ompi]
		include:
		- os: win2019
		@@ -357,6 +358,11 @@ jobs:
		- os: win2022
		image: windows-2022
		compiler: vs2022
		- os: win2022
		image: windows-2022
		shared: static
		compiler: vs2022
		parallel: serial

		defaults:
		run:

CMakeLists.txt

+3 −3

Original line number	Diff line number	Diff line
		@@ -187,6 +187,9 @@ adios_option(Campaign "Enable support for Campaigns (requires SQLite3 and ZLIB
		adios_option(AWSSDK "Enable support for S3 compatible storage using AWS SDK's S3 module" OFF)
		adios_option(Derived_Variable "Enable support for derived variables" OFF)
		adios_option(PIP "Enable support for pip packaging" OFF)

		option(ADIOS2_Blosc2_PREFER_SHARED "Prefer shared Blosc2 libraries" ON)
		mark_as_advanced(ADIOS2_Blosc2_PREFER_SHARED)
		mark_as_advanced(ADIOS2_USE_PIP)
		include(${PROJECT_SOURCE_DIR}/cmake/DetectOptions.cmake)

		@@ -217,9 +220,6 @@ if(ADIOS2_HAVE_MPI)
		add_definitions(-DOMPI_SKIP_MPICXX -DMPICH_SKIP_MPICXX)
		endif()

		cmake_dependent_option(ADIOS2_Blosc2_PREFER_SHARED "Prefer shared Blosc2 libraries"
		ON "Blosc2_shlib_available" OFF)

		#------------------------------------------------------------------------------#
		# POSIX O_DIRECT is only working for Unix in adios for now
		#------------------------------------------------------------------------------#

bindings/Python/py11Engine.cpp

+44 −4

Original line number	Diff line number	Diff line
		@@ -70,12 +70,44 @@ void Engine::Put(Variable variable, const pybind11::array &array, const Mode lau
		else
		{
		throw std::invalid_argument("ERROR: for variable " + variable.Name() +
		" numpy array type is not supported or "
		" numpy array type " + variable.Type() +
		" is not supported (found type " + ToString(type) +
		") or "
		"is not memory contiguous "
		", in call to Put\n");
		}
		}

		void Engine::Put(Variable variable, const std::vector<int64_t> &ints, const Mode launch)
		{
		helper::CheckForNullptr(m_Engine, "in call to Engine::Put list of ints");
		helper::CheckForNullptr(variable.m_VariableBase,
		"for variable, in call to Engine::Put list of ints");

		m_Engine->Put(dynamic_cast<core::Variable<int64_t> >(variable.m_VariableBase),
		reinterpret_cast<const int64_t *>(ints.data()), launch);
		}

		void Engine::Put(Variable variable, const std::vector<double> &floats, const Mode launch)
		{
		helper::CheckForNullptr(m_Engine, "in call to Engine::Put list of floats");
		helper::CheckForNullptr(variable.m_VariableBase,
		"for variable, in call to Engine::Put list of floats");

		m_Engine->Put(dynamic_cast<core::Variable<double> >(variable.m_VariableBase),
		reinterpret_cast<const double *>(floats.data()), launch);
		}

		void Engine::Put(Variable variable, const std::vector<std::complex<double>> &complexes,
		const Mode launch)
		{
		helper::CheckForNullptr(m_Engine, "in call to Engine::Put list of complexes");
		helper::CheckForNullptr(variable.m_VariableBase,
		"for variable, in call to Engine::Put list of complexes");
		m_Engine->Put(dynamic_cast<core::Variable<std::complex<double>> >(variable.m_VariableBase),
		reinterpret_cast<const std::complex<double> *>(complexes.data()), launch);
		}

		void Engine::Put(Variable variable, const std::string &string)
		{
		helper::CheckForNullptr(m_Engine, "for engine, in call to Engine::Put string");
		@@ -235,12 +267,18 @@ std::vector<std::map<std::string, std::string>> Engine::BlocksInfo(std::string &
		const size_t step) const
		{
		std::vector<std::map<std::string, std::string>> rv;
		auto &varMap = m_Engine->m_IO.GetVariables();
		auto itVariable = varMap.find(var_name);
		if (itVariable == varMap.end())
		{
		return rv;
		}

		// Grab the specified variable object and get its type string
		adios2::DataType var_type = m_Engine->GetIO().InquireVariableType(var_name);

		MinVarInfo *minBlocksInfo = nullptr;
		auto itVariable = m_Engine->m_IO.GetVariables().find(var_name);

		auto Variable = itVariable->second.get();
		minBlocksInfo = m_Engine->MinBlocksInfo(*Variable, 0);
		if (minBlocksInfo)
		@@ -261,7 +299,8 @@ std::vector<std::map<std::string, std::string>> Engine::BlocksInfo(std::string &
		{
		start_ss << ",";
		}
		start_ss << info.Start[i];
		start_ss << (minBlocksInfo->WasLocalValue ? reinterpret_cast<size_t>(info.Start)
		: info.Start[i]);
		}
		}
		info_map["Start"] = start_ss.str();
		@@ -278,7 +317,8 @@ std::vector<std::map<std::string, std::string>> Engine::BlocksInfo(std::string &
		{
		count_ss << ",";
		}
		count_ss << info.Count[i];
		count_ss << (minBlocksInfo->WasLocalValue ? reinterpret_cast<size_t>(info.Count)
		: info.Count[i]);
		}
		}
		info_map["Count"] = count_ss.str();

bindings/Python/py11Engine.h

+6 −0

Original line number	Diff line number	Diff line
		@@ -49,6 +49,12 @@ public:
		StepStatus BeginStep();

		void Put(Variable variable, const pybind11::array &array, const Mode launch = Mode::Deferred);
		void Put(Variable variable, const std::vector<int64_t> &ints,
		const Mode launch = Mode::Deferred);
		void Put(Variable variable, const std::vector<double> &doubles,
		const Mode launch = Mode::Deferred);
		void Put(Variable variable, const std::vector<std::complex<double>> &complexes,
		const Mode launch = Mode::Deferred);
		void Put(Variable variable, const std::string &string);
		void PerformPuts();
		void PerformDataWrite();

bindings/Python/py11IO.cpp

+116 −5

Original line number	Diff line number	Diff line
		@@ -101,6 +101,51 @@ Variable IO::DefineVariable(const std::string &name, const pybind11::array &arra
		return Variable(variable);
		}

		Variable IO::DefineVariable(const std::string &name, const pybind11::object &value,
		const Dims &shape, const Dims &start, const Dims &count,
		const bool isConstantDims)
		{
		helper::CheckForNullptr(m_IO, "for variable " + name + ", in call to IO::DefineVariable");
		core::VariableBase *variable = nullptr;
		const auto t = value.get_type();
		const auto ts = pybind11::str(t);
		const auto tss = pybind11::cast<std::string>(ts);
		if (pybind11::isinstance<pybind11::str>(value))
		{
		variable = &m_IO->DefineVariable<std::string>(name);
		}
		else if (pybind11::isinstance<pybind11::int_>(value))
		{
		variable = &m_IO->DefineVariable<int64_t>(name, shape, start, count, isConstantDims);
		}
		else if (pybind11::isinstance<pybind11::float_>(value))
		{
		variable = &m_IO->DefineVariable<double>(name, shape, start, count, isConstantDims);
		}
		else if (tss == "<class 'complex'>")
		{
		variable =
		&m_IO->DefineVariable<std::complex<double>>(name, shape, start, count, isConstantDims);
		}
		else if (tss == "<class 'numpy.complex64'>")
		{
		variable =
		&m_IO->DefineVariable<std::complex<float>>(name, shape, start, count, isConstantDims);
		}
		else if (tss == "<class 'numpy.complex128'>")
		{
		variable =
		&m_IO->DefineVariable<std::complex<double>>(name, shape, start, count, isConstantDims);
		}
		else
		{
		throw std::invalid_argument("ERROR: variable " + name +
		" can't be defined with an object with type " + tss +
		", in call to DefineVariable\n");
		}
		return Variable(variable);
		}

		Variable IO::InquireVariable(const std::string &name)
		{
		helper::CheckForNullptr(m_IO, "for variable " + name + ", in call to IO::InquireVariable");
		@@ -126,7 +171,6 @@ Attribute IO::DefineAttribute(const std::string &name, const pybind11::array &ar
		const std::string &variableName, const std::string separator)
		{
		helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");

		core::AttributeBase *attribute = nullptr;

		if (false)
		@@ -156,7 +200,6 @@ Attribute IO::DefineAttribute(const std::string &name, const std::string &string
		const std::string &variableName, const std::string separator)
		{
		helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");

		return Attribute(
		&m_IO->DefineAttribute<std::string>(name, stringValue, variableName, separator));
		}
		@@ -165,11 +208,78 @@ Attribute IO::DefineAttribute(const std::string &name, const std::vector<std::st
		const std::string &variableName, const std::string separator)
		{
		helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");

		return Attribute(&m_IO->DefineAttribute<std::string>(name, strings.data(), strings.size(),
		variableName, separator));
		}

		Attribute IO::DefineAttribute(const std::string &name, const std::vector<int> &ints,
		const std::string &variableName, const std::string separator)
		{
		helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");
		return Attribute(
		&m_IO->DefineAttribute<int>(name, ints.data(), ints.size(), variableName, separator));
		}

		Attribute IO::DefineAttribute(const std::string &name, const std::vector<double> &doubles,
		const std::string &variableName, const std::string separator)
		{
		helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");
		return Attribute(&m_IO->DefineAttribute<double>(name, doubles.data(), doubles.size(),
		variableName, separator));
		}

		Attribute IO::DefineAttribute(const std::string &name,
		const std::vector<std::complex<double>> &complexdoubles,
		const std::string &variableName, const std::string separator)
		{
		helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");
		return Attribute(&m_IO->DefineAttribute<std::complex<double>>(
		name, complexdoubles.data(), complexdoubles.size(), variableName, separator));
		}

		Attribute IO::DefineAttribute(const std::string &name, const pybind11::object &value,
		const std::string &variableName, const std::string separator)
		{
		helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");

		core::AttributeBase *attribute = nullptr;
		const auto t = value.get_type();
		const auto ts = pybind11::str(t);
		const auto tss = pybind11::cast<std::string>(ts);
		if (pybind11::isinstance<pybind11::int_>(value))
		{
		auto v = pybind11::cast<const int64_t>(value);
		attribute = &m_IO->DefineAttribute(name, v, variableName, separator);
		}
		else if (pybind11::isinstance<pybind11::float_>(value))
		{
		auto v = pybind11::cast<const double>(value);
		attribute = &m_IO->DefineAttribute(name, v, variableName, separator);
		}
		else if (tss == "<class 'complex'>")
		{
		auto v = pybind11::cast<const std::complex<double>>(value);
		attribute = &m_IO->DefineAttribute(name, v, variableName, separator);
		}
		else if (tss == "<class 'numpy.complex64'>")
		{
		auto v = pybind11::cast<const std::complex<float>>(value);
		attribute = &m_IO->DefineAttribute(name, v, variableName, separator);
		}
		else if (tss == "<class 'numpy.complex128'>")
		{
		auto v = pybind11::cast<const std::complex<double>>(value);
		attribute = &m_IO->DefineAttribute(name, v, variableName, separator);
		}
		else
		{
		throw std::invalid_argument("ERROR: attribute " + name +
		" can't be defined with an object with type " + tss +
		", in call to DefineAttribute\n");
		}
		return Attribute(attribute);
		}

		Attribute IO::InquireAttribute(const std::string &name, const std::string &variableName,
		const std::string separator)
		{
		@@ -234,10 +344,11 @@ std::map<std::string, Params> IO::AvailableVariables()
		return m_IO->GetAvailableVariables();
		}

		std::map<std::string, Params> IO::AvailableAttributes()
		std::map<std::string, Params> IO::AvailableAttributes(const std::string &varname,
		const std::string &separator)
		{
		helper::CheckForNullptr(m_IO, "in call to IO::AvailableAttributes");
		return m_IO->GetAvailableAttributes();
		return m_IO->GetAvailableAttributes(varname, separator);
		}

		std::string IO::VariableType(const std::string &name) const

Admin message