Wrong value read with SST and InSituMPI

Created by: jychoi-hpc

I am trying to test writing/reading a scalar value with SST and InSituMPI but getting a wrong value.

I will post my code at the end.

Basically, I get the following value (777.0) with BP4 which is correct:

$ mpirun -n 4 ./reader
 totalpe           0   777.00000000000000     
 totalpe           1   777.00000000000000     
 totalpe           2   777.00000000000000     
 totalpe           3   777.00000000000000     

But, with InSiutMPI and SST, I got the following output:

$ mpirun -n 4 ./writer : -n 4 ./reader
 totalpe           4   4.2439915819305446E-314
 totalpe           5   4.2439915819305446E-314
 totalpe           7   4.2439915819305446E-314
 totalpe           6   4.2439915819305446E-314

If someone can check my code and give some advice, I will appreciate.

My test code is as follows.

Makefile

FC=mpif90

ADIOS2_DIR=/Users/jyc/sw/adios2/devel/gnu
ADIOS2_INC=$(shell $(ADIOS2_DIR)/bin/adios2-config --fortran-flags)
ADIOS2_LIB=$(shell $(ADIOS2_DIR)/bin/adios2-config --fortran-libs)

BINS=writer reader
all: $(BINS)

writer: writer.F90
	$(FC) -g -ffree-line-length-0 $(ADIOS2_INC) -o $@ $? $(ADIOS2_LIB)

reader: reader.F90
	$(FC) -g -ffree-line-length-0 $(ADIOS2_INC) -o $@ $? $(ADIOS2_LIB)

clean:
	rm -rf $(BINS) *.o *.dSYM

writer.F90:

program writer
    use mpi
    use adios2

    implicit none

    integer(kind=8), dimension(1) :: shape_dims, start_dims, count_dims
    real, dimension(:), allocatable :: myArray
    integer :: inx, irank, isize, ierr, i, var1_type
    type(adios2_adios) :: adios
    type(adios2_io) :: io
    type(adios2_variable) :: var, var1
    type(adios2_engine) :: engine
    character(len=:), allocatable :: var1_name
    integer :: comm

    ! Launch MPI
    call MPI_Init(ierr)
    call MPI_Comm_rank(MPI_COMM_WORLD, irank, ierr)
    call MPI_Comm_size(MPI_COMM_WORLD, isize, ierr)

    call MPI_Comm_split(MPI_COMM_WORLD, 1, irank, comm, ierr)
    call MPI_Comm_rank(comm, irank, ierr)
    call MPI_Comm_size(comm, isize, ierr)

    ! Application variables
    inx = 10
    allocate( myArray(inx) )

    do i=1,inx
        myArray(i) = i-1
    end do

    ! Variable dimensions
    shape_dims(1) = isize * inx
    start_dims(1) = irank * inx
    count_dims(1) = inx

    ! Create adios handler passing the communicator, debug mode and error flag
    call adios2_init(adios, 'adios2cfg.xml', comm, adios2_debug_mode_on, ierr)

    ! Declare an IO process configuration inside adios
    call adios2_declare_io(io, adios, "group1", ierr)

    ! Defines a variable to be written in bp format
    call adios2_define_variable(var, io, 'totalpe', adios2_type_dp, ierr)
    call adios2_define_variable(var1, io, "arr1", adios2_type_real, 1, &
                                shape_dims, start_dims, count_dims, &
                                adios2_constant_dims, ierr)

    ! Open myVector_f.bp in write mode, this launches an engine
    call adios2_open(engine, io, "out.bp", adios2_mode_write, ierr)

    ! Put myArray contents to bp buffer, based on var1 metadata
    call adios2_put(engine, 'totalpe', 777D0, ierr)
    call adios2_put(engine, var1, myArray, ierr)

    ! Closes engine and deallocates it, becomes unreachable
    call adios2_close(engine, ierr)

    ! Deallocates adios and calls its destructor
    call adios2_finalize(adios, ierr)

    if( allocated(myArray) ) deallocate(myArray)
    if( allocated(var1_name) ) deallocate(var1_name)

    call MPI_Finalize(ierr)

end program writer

reader.F90:

program reader
    use mpi
    use adios2
    implicit none

    integer(kind=8), dimension(2) :: sel_start, sel_count
    real, dimension(:,:), allocatable :: data
    integer :: i, j, inx, iny, irank, isize, ierr
    real*8 :: totalpe;
    integer :: comm

    ! adios2 handlers
    type(adios2_adios):: adios
    type(adios2_io):: io
    type(adios2_variable):: var
    type(adios2_engine):: engine

    ! Launch MPI
    call MPI_Init(ierr)
    call MPI_Comm_rank(MPI_COMM_WORLD, irank, ierr)
    call MPI_Comm_size(MPI_COMM_WORLD, isize, ierr)

    call MPI_Comm_split(MPI_COMM_WORLD, 2, irank, comm, ierr)

    call adios2_init(adios, 'adios2cfg.xml', comm, adios2_debug_mode_on, ierr)

    call adios2_declare_io(io, adios, "group1", ierr)
    call adios2_open(engine, io, "out.bp", adios2_mode_read, ierr)
    call adios2_get(engine, "totalpe", totalpe, ierr)
    call adios2_close(engine, ierr)

    print *, 'totalpe', irank, totalpe

    call adios2_finalize(adios, ierr)
    call MPI_Finalize(ierr)

end program reader

adios2cfg.xml:

<?xml version="1.0"?>
<adios-config>
    <io name="group1">
        <engine type="InSituMPI">
            <!--
            -->
            <parameter key="QueueLimit" value="1"/>
            <parameter key="node-local" value="Off"/>
            <parameter key="DataTransport" value="RDMA"/>
            <parameter key="Profile" value="Off"/>
            <parameter key="OpenTimeoutSecs" value="1000"/>
            <parameter key="BeginStepPollingFrequencySecs" value="1.0"/>
            <parameter key="DataTransport" value="WAN"/>

        </engine>
    </io>
</adios-config>