Unverified Commit 18205cef authored by Eisenhauer, Greg's avatar Eisenhauer, Greg Committed by GitHub
Browse files

Several fixes for ASAN issues (#3959) 

* BP5 Min/Max Cleanup
* mod attr fix
* Testing leak
* overall adios suppression file
* Fix leak on array of strings
* Clarify attribute handling
* minblocks leaks in bpls
* Leak in C bindings
* Release IOs in test
* Finalize adios to free memory
* Only ignore unfixed asan issues
parent 17c2bd0d

CTestConfig.cmake

+32 −172
Original line number Diff line number Diff line
@@ -14,175 +14,35 @@ set(MEMORYCHECK_SUPPRESSIONS_FILE ${CMAKE_SOURCE_DIR}/scripts/dashboard/nightly/ 


# Ignore tests that are currently failing, remove tests here as they are fixed

list(APPEND CTEST_CUSTOM_MEMCHECK_IGNORE 

  Bindings.C.ADIOS2_C_API.ADIOS2BPWriteTypes.Serial

  Bindings.C.BPWriteReadMultiblockCC.ZeroSizeBlocks.Serial

  Engine.BP.*/BPAppendAfterStepsP.Test/*.BP5.Serial

  Engine.BP.*/BPChangingShapeWithinStep.MultiBlock/*.BP5.Serial

  Engine.BP.*/BPParameterSelectStepsP.Read/*.BP5.Serial

  Engine.BP.*/BPReadMultithreadedTestP.ReadFile/*.BP5.Serial

  Engine.BP.*/BPStepsFileGlobalArrayParameters.EveryOtherStep/*.BP5.Serial

  Engine.BP.*/BPStepsFileGlobalArrayReaders.EveryStep/*.BP5.Serial

  Engine.BP.*/BPStepsFileGlobalArrayReaders.NewVarPerStep/*.BP5.Serial

  Engine.BP.*/BPStepsFileLocalArrayParameters.EveryOtherStep/*.BP5.Serial

  Engine.BP.*/BPStepsFileLocalArrayReaders.EveryStep/*.BP5.Serial

  Engine.BP.*/BPStepsFileLocalArrayReaders.NewVarPerStep/*.BP5.Serial

  Engine.BP.BPChangingShape.BPWriteReadShape2D.BP5.Serial

  Engine.BP.BPLargeMetadata.ManyLongStrings.BP5.Serial

  Engine.BP.BPWriteAppendReadTestADIOS2.ADIOS2BPWriteAppendRead2D2x4.BP5.Serial

  Engine.BP.BPWriteAppendReadTestADIOS2.ADIOS2BPWriteAppendReadAggregate.BP5.Serial

  Engine.BP.BPWriteAppendReadTestADIOS2.ADIOS2BPWriteAppendReadVaryingAggregation.BP5.Serial

  Engine.BP.BPWriteMultiblockReadTest.ADIOS2BPWriteMultiblockRead1D8.BP5.Serial

  Engine.BP.BPWriteMultiblockReadTest.ADIOS2BPWriteMultiblockRead2D2x4.BP5.Serial

  Engine.BP.BPWriteMultiblockReadTest.ADIOS2BPWriteMultiblockRead2D4x2.BP5.Serial

  Engine.BP.BPWriteMultiblockReadTest.ADIOS2BPWriteRead1D8ZeroBlock.BP5.Serial

  Engine.BP.BPWriteReadAsStreamTestADIOS2.ReaderWriterDefineVariable.BP5.Serial

  Engine.BP.BPWriteReadAttributes.BPWriteReadSingleTypesVar.BP5.Serial

  Engine.BP.BPWriteReadAttributes.WriteReadArrayTypes.BP5.Serial

  Engine.BP.BPWriteReadAttributes.WriteReadArrayTypesVar.BP5.Serial

  Engine.BP.BPWriteReadAttributes.WriteReadSingleTypes.BP5.Serial

  Engine.BP.BPWriteReadAttributes.WriteReadStreamVarp.BP5.Serial

  Engine.BP.BPWriteReadAttributeTestMultirank.ADIOS2BPWriteReadArrayTypes.BP5.Serial

  Engine.BP.BPWriteReadBlockInfo.BPWriteReadBlockInfo1D8_C.BP3.Serial

  Engine.BP.BPWriteReadBlockInfo.BPWriteReadBlockInfo1D8_C.BP4.Serial

  Engine.BP.BPWriteReadBlockInfo.BPWriteReadBlockInfo1D8_C.BP5.Serial

  Engine.BP.BPWriteReadBlockInfo.BPWriteReadBlockInfo1D8.BP5.Serial

  Engine.BP.BPWriteReadBlockInfo.BPWriteReadBlockInfo2D2x4.BP5.Serial

  Engine.BP.BPWriteReadLocalVariables.ADIOS2BPWriteReadLocal1DAllSteps.BP5.Serial

  Engine.BP.BPWriteReadLocalVariables.ADIOS2BPWriteReadLocal1DBlockInfo.BP5.Serial

  Engine.BP.BPWriteReadLocalVariables.ADIOS2BPWriteReadLocal2DChangeCount.BP3.Serial

  Engine.BP.BPWriteReadLocalVariables.ADIOS2BPWriteReadLocal2DChangeCount.BP4.Serial

  Engine.BP.BPWriteReadLocalVariables.ADIOS2BPWriteReadLocal2DChangeCount.BP5.Serial

  Engine.BP.BPWriteReadLocalVariablesSel.BPWriteReadLocal1DAllStepsSel.BP5.Serial

  Engine.BP.BPWriteReadLocalVariablesSelHighLevel.BPWriteReadLocal1DAllStepsSel.BP5.Serial

  Engine.BP.BPWriteReadMultiblockTest.ADIOS2BPWriteReadMultiblock1D8.BP5.Serial

  Engine.BP.BPWriteReadMultiblockTest.ADIOS2BPWriteReadMultiblock2D2x4.BP5.Serial

  Engine.BP.BPWriteReadMultiblockTest.ADIOS2BPWriteReadMultiblock2D4x2.BP5.Serial

  Engine.BP.BPWriteReadMultiblockTest.MultiblockNullBlocks.BP5.Serial

  Engine.BP.BPWriteReadMultiblockTest.MultiblockPerformDataWrite.BP5.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead10D2x2.Async.BP5.EWS.Guided.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead10D2x2.Async.BP5.EWS.Naive.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead10D2x2.Async.BP5.TLS.Guided.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead10D2x2.Async.BP5.TLS.Naive.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead10D2x2.BP5.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead1D8.Async.BP5.EWS.Guided.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead1D8.Async.BP5.EWS.Naive.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead1D8.Async.BP5.TLS.Guided.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead1D8.Async.BP5.TLS.Naive.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead1D8.BP5.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D2x4.Async.BP5.EWS.Guided.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D2x4.Async.BP5.EWS.Naive.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D2x4.Async.BP5.TLS.Guided.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D2x4.Async.BP5.TLS.Naive.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D2x4.BP5.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.Async.BP5.EWS.Guided.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.Async.BP5.EWS.Naive.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.Async.BP5.TLS.Guided.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.Async.BP5.TLS.Naive.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.BP5.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.Async.BP5.EWS.Guided.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.Async.BP5.EWS.Naive.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.Async.BP5.TLS.Guided.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.Async.BP5.TLS.Naive.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.BP5.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2.Async.BP5.EWS.Guided.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2.Async.BP5.EWS.Naive.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2.Async.BP5.TLS.Guided.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2.Async.BP5.TLS.Naive.Serial

  Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2.BP5.Serial

  Engine.BP.BPWriteReadTestADIOS2.GetDeferredInClose.Async.BP5.EWS.Guided.Serial

  Engine.BP.BPWriteReadTestADIOS2.GetDeferredInClose.Async.BP5.EWS.Naive.Serial

  Engine.BP.BPWriteReadTestADIOS2.GetDeferredInClose.Async.BP5.TLS.Guided.Serial

  Engine.BP.BPWriteReadTestADIOS2.GetDeferredInClose.Async.BP5.TLS.Naive.Serial

  Engine.BP.BPWriteReadTestADIOS2.GetDeferredInClose.BP5.Serial

  Engine.BP.BPWriteReadTestADIOS2.ReadStartCount.Async.BP5.EWS.Guided.Serial

  Engine.BP.BPWriteReadTestADIOS2.ReadStartCount.Async.BP5.EWS.Naive.Serial

  Engine.BP.BPWriteReadTestADIOS2.ReadStartCount.Async.BP5.TLS.Guided.Serial

  Engine.BP.BPWriteReadTestADIOS2.ReadStartCount.Async.BP5.TLS.Naive.Serial

  Engine.BP.BPWriteReadTestADIOS2.ReadStartCount.BP5.Serial

  Engine.BP.BPWriteReadTestADIOS2fstream.ADIOS2BPWriteRead1D8.BP5.Serial

  Engine.BP.BPWriteReadTestADIOS2fstream.ADIOS2BPWriteRead2D2x4.BP5.Serial

  Engine.BP.BPWriteReadTestADIOS2fstream.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.BP5.Serial

  Engine.BP.BPWriteReadTestADIOS2fstream.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.BP5.Serial

  Engine.BP.BPWriteReadTestADIOS2fstream.ADIOS2BPWriteRead2D4x2.BP5.Serial

  Engine.BP.BPWriteReadTestADIOS2stdio.ADIOS2BPWriteRead1D8.BP5.Serial

  Engine.BP.BPWriteReadTestADIOS2stdio.ADIOS2BPWriteRead2D2x4.BP5.Serial

  Engine.BP.BPWriteReadTestADIOS2stdio.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.BP5.Serial

  Engine.BP.BPWriteReadTestADIOS2stdio.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.BP5.Serial

  Engine.BP.BPWriteReadTestADIOS2stdio.ADIOS2BPWriteRead2D4x2.BP5.Serial

  Engine.BP.BPWriteReadVector.ADIOS2BPWriteRead1D8.BP5.Serial

  Engine.BP.BPWriteReadVector.ADIOS2BPWriteRead2D2x4.BP5.Serial

  Engine.BP.BPWriteReadVector.ADIOS2BPWriteRead2D4x2.BP5.Serial

  Engine.BP.BPWriteReadVector.ADIOS2BPWriteReadVector2D4x2_MultiSteps.BP5.Serial

  Engine.BPEngineTest.ZfpComplex.Serial

  Engine.SST.SstWriteFails.InvalidBeginStep.Serial

  Engine.SST.SstWriteFails.InvalidPut.Serial

  Engine.Staging.TestThreads.Basic.SST.BP.Serial

  Engine.Staging.TestThreads.Basic.SST.FFS.Serial

  Interface.ADIOS2_CXX11_API_Selection.MemorySelectionComplex.BPfile.Serial

  Interface.ADIOS2_CXX11_API_Selection.MemorySelectionReadStart.BPfile.Serial

  Remote.BPWriteMemorySelectionRead.FileRemote

  Remote.BPWriteMemorySelectionRead.GetRemote

  Remote.BPWriteReadADIOS2stdio.GetRemote

  remoteServerCleanup

  remoteServerSetup

  Staging.1x1.Attrs.BP5

  Staging.1x1.Attrs.CommMin.BP.SST

  Staging.1x1.Attrs.CommMin.BP5.SST

  Staging.1x1.CommMin.BP.SST

  Staging.1x1.CommMin.BP5.SST

  Staging.1x1.ForcePreload.CommMin.BP.SST

  Staging.1x1.ForcePreload.CommMin.BP5.SST

Staging.TimeoutReader.1x1.CommMin.BP5.SST

Staging.1x1Struct.BP5

Staging.WriteMemorySelectionRead.1x1.CommMin.BP.SST

Staging.1x1.Local2.CommMin.BP.SST

  Staging.1x1.Local2.CommMin.BP5.SST

  Staging.1x1.LocalMultiblock.CommMin.BP.SST

Staging.OnDemandSingle.1x1.CommMin.BP5.SST

Staging.AllToAllDistribution.1x1x3.CommMin.BP5.SST

Staging.RoundRobinDistribution.1x1x3.CommMin.BP5.SST

Staging.1x1.LocalMultiblock.CommMin.BP5.SST

  Staging.1x1.ModAttrs.BP5

  Staging.1x1.ModAttrs.CommMin.BP.SST

  Staging.1x1.ModAttrs.CommMin.BP5.SST

  Staging.1x1.NoPreload.CommMin.BP.SST

  Staging.1x1.NoPreload.CommMin.BP5.SST

  Staging.1x1.SstRUDP.CommMin.BP.SST

Staging.1x1.SstRUDP.CommMin.BP5.SST

  Staging.1x1Joined.BP5

  Staging.1x1Joined.CommMin.BP5.SST

  Staging.1x1LockGeometry.CommMin.BP.SST

Staging.WriteMemorySelectionRead.1x1.CommMin.BP5.SST

Staging.1x1LockGeometry.CommMin.BP5.SST

  Staging.1x1Struct.BP5

  Staging.1x1Struct.CommMin.BP5.SST

  Staging.1x1VarDestruction.CommMin.BP.SST

  Staging.1x1VarDestruction.CommMin.BP5.SST

  Staging.AllToAllDistribution.1x1x3.CommMin.BP.SST

  Staging.AllToAllDistribution.1x1x3.CommMin.BP5.SST

  Staging.DiscardWriter.1x1.CommMin.BP.SST

Staging.1x1.ForcePreload.CommMin.BP5.SST

Staging.1x1.NoPreload.CommMin.BP5.SST

Staging.DiscardWriter.1x1.CommMin.BP5.SST

  Staging.LatestReader.1x1.CommMin.BP.SST

  Staging.LatestReader.1x1.CommMin.BP5.SST

  Staging.LatestReaderHold.1x1.CommMin.BP.SST

Staging.LatestReaderHold.1x1.CommMin.BP5.SST

  Staging.OnDemandSingle.1x1.CommMin.BP.SST

  Staging.OnDemandSingle.1x1.CommMin.BP5.SST

  Staging.RoundRobinDistribution.1x1x3.CommMin.BP.SST

  Staging.RoundRobinDistribution.1x1x3.CommMin.BP5.SST

  Staging.TimeoutReader.1x1.CommMin.BP.SST

  Staging.TimeoutReader.1x1.CommMin.BP5.SST

  Staging.WriteMemorySelectionRead.1x1.CommMin.BP.SST

  Staging.WriteMemorySelectionRead.1x1.CommMin.BP5.SST

  Staging.ZFPCompression.1x1.CommMin.BP.SST

  Staging.ZFPCompression.1x1.CommMin.BP5.SST

  Staging.ZFPCompression.3x5.CommMin.BP.SST

Staging.LatestReader.1x1.CommMin.BP5.SST

Engine.BP.BPWriteReadAsStreamTestADIOS2.ReaderWriterDefineVariable.BP5.Serial

Staging.ZFPCompression.3x5.CommMin.BP5.SST

  Utils.ChangingShape.AlternatingStepsAndChangingShapeVar.Dump

  Utils.ChangingShape.AlternatingStepsVarSelection.Dump

  Utils.ChangingShape.ChangingShapeVarOneStep.Dump

  Utils.ChangingShape.Dump

  Utils.ChangingShape.FixedShapeVarTooManySteps.Dump

  Utils.ChangingShape.Screen

  Utils.CWriter

  Utils.CWriter.Bpls.Al.Dump

  Utils.CWriter.Bpls.h.Dump

  Utils.CWriter.Bpls.la.Dump

  Utils.CWriter.Bpls.la.Screen

  Utils.CWriter.Bpls.ldDav.Dump

  Utils.CWriter.Bpls.ldDavvv.Dump

  Utils.CWriter.Bpls.ldvarI16.Dump

Staging.ZFPCompression.1x1.CommMin.BP5.SST

Staging.1x1Joined.CommMin.BP5.SST

Staging.1x1Struct.CommMin.BP5.SST

Staging.1x1.Local2.CommMin.BP5.SST

Staging.1x1VarDestruction.CommMin.BP5.SST

Staging.1x1.ModAttrs.CommMin.BP5.SST

Staging.1x1.Attrs.CommMin.BP5.SST

Staging.1x1.CommMin.BP5.SST

Engine.SST.SstWriteFails.InvalidBeginStep.Serial

remoteServerCleanup

Remote.BPWriteMemorySelectionRead.FileRemote

Remote.BPWriteMemorySelectionRead.GetRemote

Remote.BPWriteReadADIOS2stdio.GetRemote

)

bindings/C/adios2/c/adios2_c_engine.cpp

+3 −2
Original line number Diff line number Diff line
@@ -699,8 +699,7 @@ adios2_varinfo *adios2_inquire_blockinfo(adios2_engine *engine, adios2_variable 
                b[i].MinUnion.uint64 = 0;

                // minBlocksInfo->BlocksInfo[i].MaxUnion;

                b[i].MaxUnion.uint64 = 0;

                b[i].Value.str = (char *)malloc(blocksInfo[i].Value.size() + 1);

                std::strcpy(b[i].Value.str, blocksInfo[i].Value.data());

                b[i].Value.str = (char *)blocksInfo[i].Value.data();

            };

        }

#define declare_template_instantiation(T)                                                          \
@@ -749,6 +748,8 @@ void adios2_free_blockinfo(adios2_varinfo *data_blocks) 
            free(data_blocks->BlocksInfo[i].Start);

            free(data_blocks->BlocksInfo[i].Count);

        }

        if (data_blocks->Shape)

            free(data_blocks->Shape);

        free(data_blocks->BlocksInfo);

        free(data_blocks);

    }

scripts/ci/cmake/adios-asan.supp

0 → 100644
+2 −0
Original line number Diff line number Diff line 
leak:ps_make_timer_name_

leak:ibv_get_device_list
 
 No newline at end of file

scripts/ci/cmake/ci-fedora-asan.cmake

+1 −1
Original line number Diff line number Diff line
@@ -4,7 +4,7 @@ set(ENV{CC} clang) 
set(ENV{CXX} clang++)

set(ASAN_FLAGS "-fsanitize=address -fno-omit-frame-pointer -pthread -mllvm -asan-use-private-alias=1 -Wno-unused-command-line-argument")

set(ENV{ASAN_OPTIONS} "use_odr_indicator=1")

set(ENV{LSAN_OPTIONS} "suppressions=$ENV{CI_SOURCE_DIR}/thirdparty/perfstubs/perfstubs.supp")

set(ENV{LSAN_OPTIONS} "suppressions=$ENV{CI_SOURCE_DIR}/scripts/ci/cmake/adios-asan.supp")

set(ENV{CFLAGS}   "${ASAN_FLAGS}")

set(ENV{CXXFLAGS} "${ASAN_FLAGS}")

source/adios2/common/ADIOSTypes.cpp

+21 −0
Original line number Diff line number Diff line
@@ -390,6 +390,27 @@ int TypeElementSize(DataType adiosvartype) 
    }

}



bool TypeHasMinMax(DataType adiosvartype)

{

    switch (adiosvartype)

    {

    case DataType::UInt8:

    case DataType::Int8:

    case DataType::UInt16:

    case DataType::Int16:

    case DataType::UInt32:

    case DataType::Int32:

    case DataType::UInt64:

    case DataType::Int64:

    case DataType::Float:

    case DataType::Double:

    case DataType::LongDouble:

        return true;

    default:

        return false;

    }

}



static void PrintMBI(std::ostream &os, const MinBlockInfo &blk, int Dims)

{

    os << "Writer: " << blk.WriterID << ", Blk: " << blk.BlockID << ", Start: {";