Fix a problem with FFS marshaling that occurs when you don't write the same... (1d64c264) · Commits · Podhorszki, Norbert / ADIOS2

source/adios2/toolkit/sst/cp/ffs_marshal.c

+39 −4

Original line number	Diff line number	Diff line
		@@ -320,6 +320,8 @@ struct FFSMetadataInfoStruct
		size_t DataBlockSize;
		};

		static int FFSBitfieldTest(struct FFSMetadataInfoStruct *MBase, int Bit);

		static void InitMarshalData(SstStream Stream)
		{
		struct FFSWriterMarshalBase *Info =
		@@ -804,9 +806,11 @@ static void DecodeAndPrepareData(SstStream Stream, int Writer)
		(ArrayRec )((char )BaseData + FieldList[i].field_offset);
		const char *ArrayName = FieldList[i + 1].field_name + 4;
		FFSVarRec VarRec = LookupVarByName(Stream, ArrayName);
		if (VarRec) {
		VarRec->PerWriterIncomingData[Writer] = data_base->Array;
		VarRec->PerWriterIncomingSize[Writer] = data_base->ElemCount;
		VarRec->PerWriterDataFieldDesc[Writer] = &FieldList[i + 1];
		}
		i += 2;
		}
		}
		@@ -1249,6 +1253,8 @@ extern void SstFFSWriterEndStep(SstStream Stream, size_t Timestep)
		FMfree_var_rec_elements(Info->MetaFormat, Stream->M);
		MBase->BitField = tmp;
		FMfree_var_rec_elements(Info->DataFormat, Stream->D);
		memset(Stream->M, 0, Stream->MetadataSize);
		memset(Stream->D, 0, Stream->DataSize);

		// Call SstInternalProvideStep with Metadata block, Data block and (any new)
		// formatID and formatBody
		@@ -1420,7 +1426,7 @@ static void BuildVarList(SstStream Stream, TSMetadataMsg MetaData,
		while (strncmp(FieldList->field_name, "DataBlockSize", 8) == 0)
		FieldList++;
		int i = 0;

		int j = 0;
		while (FieldList[i].field_name)
		{
		void field_data = (char )BaseData + FieldList[i].field_offset;
		@@ -1431,6 +1437,12 @@ static void BuildVarList(SstStream Stream, TSMetadataMsg MetaData,
		char *Type;
		FFSVarRec VarRec = NULL;
		int ElementSize;
		if (!FFSBitfieldTest(BaseData, j)) {
		/* only work with fields that were written */
		i += 4;
		j++;
		continue;
		}
		BreakdownArrayName(FieldList[i].field_name, &ArrayName, &Type,
		&ElementSize);
		if (WriterRank != 0)
		@@ -1473,6 +1485,12 @@ static void BuildVarList(SstStream Stream, TSMetadataMsg MetaData,
		/* simple field */
		char *FieldName = strdup(FieldList[i].field_name + 4); // skip SST_
		FFSVarRec VarRec = NULL;
		if (!FFSBitfieldTest(BaseData, j)) {
		/* only work with fields that were written */
		i++;
		j++;
		continue;
		}
		if (WriterRank != 0)
		{
		VarRec = LookupVarByName(Stream, FieldName);
		@@ -1490,6 +1508,8 @@ static void BuildVarList(SstStream Stream, TSMetadataMsg MetaData,
		VarRec->PerWriterDataFieldDesc[WriterRank] = NULL;
		i++;
		}
		/* real variable count is in j, i tracks the entries in the metadata */
		j++;
		}
		}

		@@ -1525,6 +1545,21 @@ static void FFSBitfieldSet(struct FFSMetadataInfoStruct *MBase, int Bit)
		MBase->BitField[Element] \|= (1 << ElementBit);
		}

		static int FFSBitfieldTest(struct FFSMetadataInfoStruct *MBase, int Bit)
		{
		int Element = Bit / 64;
		int ElementBit = Bit % 64;
		if (Element >= MBase->BitFieldCount)
		{
		MBase->BitField =
		realloc(MBase->BitField, sizeof(size_t) * (Element + 1));
		memset(MBase->BitField + MBase->BitFieldCount * sizeof(size_t), 0,
		(Element - MBase->BitFieldCount + 1) * sizeof(size_t));
		MBase->BitFieldCount = Element + 1;
		}
		return ((MBase->BitField[Element] & (1 << ElementBit)) == (1 << ElementBit));
		}

		extern void SstFFSSetZFPParams(SstStream Stream, attr_list Attrs)
		{
		if (Stream->WriterMarshalData)

testing/adios2/engine/sst/CMakeLists.txt

+10 −0

Original line number	Diff line number	Diff line
		@@ -4,6 +4,7 @@
		#------------------------------------------------------------------------------#

		add_executable(TestSstWrite TestSstWrite.cpp)
		add_executable(TestSstWriteModes TestSstWriteModes.cpp)
		add_executable(TestSstRead TestSstRead.cpp)
		add_executable(TestSstServer TestSstServer.cpp)
		add_executable(TestSstClient TestSstClient.cpp)
		@@ -26,11 +27,13 @@ endif()
		# Workaround for multiple versions of FindSst
		if(SST_INCLUDE_DIRS)
		target_include_directories(TestSstWrite PRIVATE ${SST_INCLUDE_DIRS})
		target_include_directories(TestSstWriteModes PRIVATE ${SST_INCLUDE_DIRS})
		target_include_directories(TestSstRead PRIVATE ${SST_INCLUDE_DIRS})
		target_include_directories(TestSstServer PRIVATE ${SST_INCLUDE_DIRS})
		target_include_directories(TestSstClient PRIVATE ${SST_INCLUDE_DIRS})
		endif()
		target_link_libraries(TestSstWrite adios2 gtest_interface ${Sst_LIBRARY})
		target_link_libraries(TestSstWriteModes adios2 gtest_interface ${Sst_LIBRARY})
		target_link_libraries(TestSstRead adios2 gtest_interface ${Sst_LIBRARY})
		target_link_libraries(TestSstServer adios2 gtest_interface ${Sst_LIBRARY})
		target_link_libraries(TestSstClient adios2 gtest_interface ${Sst_LIBRARY})
		@@ -39,6 +42,7 @@ target_link_libraries(TestSstWriterFails adios2 gtest)

		if(ADIOS2_HAVE_MPI)
		target_link_libraries(TestSstWrite MPI::MPI_C)
		target_link_libraries(TestSstWriteModes MPI::MPI_C)
		target_link_libraries(TestSstRead MPI::MPI_C)
		target_link_libraries(TestSstServer MPI::MPI_C)
		target_link_libraries(TestSstClient MPI::MPI_C)
		@@ -81,6 +85,12 @@ add_test(
		-nw 1 -nr 0 -v -p TestSst -arg "RendezvousReaderCount:0,QueueLimit:3,QueueFullPolicy:discard")
		set_tests_properties(ADIOSSstTest.NoReaderNoWait PROPERTIES TIMEOUT 30 RUN_SERIAL 1)

		add_test(
		NAME ADIOSSstTest.Modes
		COMMAND ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/run_staging_test
		-nw 1 -nr 1 -v -w TestSstWriteModes -r TestSstRead)
		set_tests_properties(ADIOSSstTest.Modes PROPERTIES TIMEOUT 30 RUN_SERIAL 1)

		if(ADIOS2_HAVE_Fortran)
		add_test(
		NAME ADIOSSstTest.FtoC_1x1

testing/adios2/engine/sst/TestData.h

+56 −49

Original line number	Diff line number	Diff line
		@@ -76,7 +76,7 @@ void generateSstTestData(int step, int rank, int size)
		}
		}

		int validateSstTestData(int start, int length, int step)
		int validateSstTestData(int start, int length, int step, int missing_end_data)
		{
		int failures = 0;
		if (in_scalar_R64 != 1.5 * (step + 1))
		@@ -135,37 +135,42 @@ int validateSstTestData(int start, int length, int step)
		<< "](global[" << i + start << "])" << std::endl;
		failures++;
		}
		if (!missing_end_data)
		{
		if ((in_C32[i].imag() != (float)((i + start) * 10 + step)) \|\|
		(in_C32[i].real() != -(float)((i + start) * 10 + step)))
		{
		std::cout << "Expected [" << (float)((i + start) * 10 + step)
		<< ", " << -(float)((i + start) * 10 + step) << "], got "
		<< in_C32[i] << " for in_C32[" << i << "](global["
		<< i + start << "])" << std::endl;
		<< ", " << -(float)((i + start) * 10 + step)
		<< "], got " << in_C32[i] << " for in_C32[" << i
		<< "](global[" << i + start << "])" << std::endl;
		failures++;
		}
		if ((in_C64[i].imag() != (double)((i + start) * 10 + step)) \|\|
		(in_C64[i].real() != (-(double)((i + start) * 10 + step))))
		{
		std::cout << "Expected [" << (double)((i + start) * 10 + step)
		<< ", " << -(double)((i + start) * 10 + step) << "], got "
		<< in_C64[i] << " for in_C64[" << i << "](global["
		<< i + start << "])" << std::endl;
		<< ", " << -(double)((i + start) * 10 + step)
		<< "], got " << in_C64[i] << " for in_C64[" << i
		<< "](global[" << i + start << "])" << std::endl;
		failures++;
		}
		if (in_R64_2d[2 * i] != (double)((i + start) * 10 + step))
		{
		std::cout << "Expected " << (double)((i + start) * 10 + step)
		<< ", got " << in_R64_2d[i] << " for in_R64_2d[" << i
		<< "][0](global[" << i + start << "][0])" << std::endl;
		<< "][0](global[" << i + start << "][0])"
		<< std::endl;
		failures++;
		}
		if (in_R64_2d[2 * i + 1] != (double)(10000 + (i + start) * 10 + step))
		if (in_R64_2d[2 * i + 1] !=
		(double)(10000 + (i + start) * 10 + step))
		{
		std::cout << "Expected "
		<< (double)(10000 + (i + start) * 10 + step) << ", got "
		<< in_R64_2d[i] << " for in_R64_2d[" << i
		<< "][1](global[" << i + start << "][1])" << std::endl;
		<< (double)(10000 + (i + start) * 10 + step)
		<< ", got " << in_R64_2d[i] << " for in_R64_2d[" << i
		<< "][1](global[" << i + start << "][1])"
		<< std::endl;
		failures++;
		}
		if (in_R64_2d_rev[i] != (double)((i + start) * 10 + step))
		@@ -180,12 +185,14 @@ int validateSstTestData(int start, int length, int step)
		(double)(10000 + (i + start) * 10 + step))
		{
		std::cout << "Expected "
		<< (double)(10000 + (i + start) * 10 + step) << ", got "
		<< in_R64_2d_rev[i + length] << " for in_R64_2d_rev[1]["
		<< i << "](global[1][" << i + start << "])" << std::endl;
		<< (double)(10000 + (i + start) * 10 + step)
		<< ", got " << in_R64_2d_rev[i + length]
		<< " for in_R64_2d_rev[1][" << i << "](global[1]["
		<< i + start << "])" << std::endl;
		failures++;
		}
		}
		}
		return failures;
		}

testing/adios2/engine/sst/TestSstClient.cpp

+1 −1

Original line number	Diff line number	Diff line
		@@ -235,7 +235,7 @@ TEST_F(SstReadTest, ADIOS2SstRead)
		{
		engine.EndStep();

		EXPECT_EQ(validateSstTestData(myStart, myLength, t), 0);
		EXPECT_EQ(validateSstTestData(myStart, myLength, t, 0), 0);
		write_times.push_back(write_time);
		}
		catch (...)

testing/adios2/engine/sst/TestSstRead.cpp

+44 −27

Original line number	Diff line number	Diff line
		@@ -137,27 +137,36 @@ TEST_F(SstReadTest, ADIOS2SstRead1D8)
		ASSERT_EQ(var_r64.Shape()[0], writerSize * Nx);

		auto var_c32 = io.InquireVariable<std::complex<float>>("c32");
		auto var_c64 = io.InquireVariable<std::complex<double>>("c64");
		auto var_r64_2d = io.InquireVariable<double>("r64_2d");
		auto var_r64_2d_rev = io.InquireVariable<double>("r64_2d_rev");
		if (var_c32)
		{
		EXPECT_TRUE(var_c32);
		ASSERT_EQ(var_c32.ShapeID(), adios2::ShapeID::GlobalArray);
		ASSERT_EQ(var_c32.Shape()[0], writerSize * Nx);

		auto var_c64 = io.InquireVariable<std::complex<double>>("c64");
		EXPECT_TRUE(var_c64);
		ASSERT_EQ(var_c64.ShapeID(), adios2::ShapeID::GlobalArray);
		ASSERT_EQ(var_c64.Shape()[0], writerSize * Nx);

		auto var_r64_2d = io.InquireVariable<double>("r64_2d");
		EXPECT_TRUE(var_r64_2d);
		ASSERT_EQ(var_r64_2d.ShapeID(), adios2::ShapeID::GlobalArray);
		ASSERT_EQ(var_r64_2d.Shape()[0], writerSize * Nx);
		ASSERT_EQ(var_r64_2d.Shape()[1], 2);

		auto var_r64_2d_rev = io.InquireVariable<double>("r64_2d_rev");
		EXPECT_TRUE(var_r64_2d_rev);
		ASSERT_EQ(var_r64_2d_rev.ShapeID(), adios2::ShapeID::GlobalArray);
		ASSERT_EQ(var_r64_2d_rev.Shape()[0], 2);
		ASSERT_EQ(var_r64_2d_rev.Shape()[1], writerSize * Nx);

		}
		else
		{
		EXPECT_FALSE(var_c32);
		EXPECT_FALSE(var_c64);
		EXPECT_FALSE(var_r64_2d);
		EXPECT_FALSE(var_r64_2d_rev);
		}
		auto var_time = io.InquireVariable<int64_t>("time");
		EXPECT_TRUE(var_time);
		ASSERT_EQ(var_time.ShapeID(), adios2::ShapeID::GlobalArray);
		@@ -192,9 +201,13 @@ TEST_F(SstReadTest, ADIOS2SstRead1D8)

		var_r32.SetSelection(sel);
		var_r64.SetSelection(sel);
		if (var_c32)
		var_c32.SetSelection(sel);
		if (var_c64)
		var_c64.SetSelection(sel);
		if (var_r64_2d)
		var_r64_2d.SetSelection(sel2);
		if (var_r64_2d_rev)
		var_r64_2d_rev.SetSelection(sel3);

		var_time.SetSelection(sel_time);
		@@ -218,15 +231,19 @@ TEST_F(SstReadTest, ADIOS2SstRead1D8)

		engine.Get(var_r32, in_R32.data());
		engine.Get(var_r64, in_R64.data());
		if (var_c32)
		engine.Get(var_c32, in_C32.data());
		if (var_c64)
		engine.Get(var_c64, in_C64.data());
		if (var_r64_2d)
		engine.Get(var_r64_2d, in_R64_2d.data());
		if (var_r64_2d_rev)
		engine.Get(var_r64_2d_rev, in_R64_2d_rev.data());
		std::time_t write_time;
		engine.Get(var_time, (int64_t *)&write_time);
		engine.EndStep();

		EXPECT_EQ(validateSstTestData(myStart, myLength, t), 0);
		EXPECT_EQ(validateSstTestData(myStart, myLength, t, !var_c32), 0);
		write_times.push_back(write_time);
		++t;
		}

Admin message