Commit c6f3874c authored by EVPath Upstream's avatar EVPath Upstream Committed by Eisenhauer, Greg
Browse files

EVPath 2018-12-03 (91608041) 

Code extracted from:

    https://github.com/GTkorvo/EVPath.git

at commit 91608041b718b769ed16eddaccb12ccc0c4ed9a5 (master).

Upstream Shortlog
-----------------
parent 2e74ff24

.gitignore

+2 −0
Original line number Diff line number Diff line 
# vi temp files

*.s??



# /

/*.exp

.indent.pro

+1 −1
Original line number Diff line number Diff line 
-nbad -bap -nbc -br -c33  -cd33  -ncdb  -ce  -ci4

-nbad -bap -nbc -br -c33  -cd33  -ncdb  -ce  -ci4 -brs

-cli0  -cp33  -d0  -di1	-fc1  -fca -i4 -ip0 -l75 -lp

-npcs -psl -sc -nsob -nss -ts8 -TIOFile -TIOFormat -TIOConversionPtr 

-TIOFieldList -TIORecordType -TIOFieldPtr

CMakeLists.txt

+110 −15
Original line number Diff line number Diff line 
cmake_minimum_required(VERSION 3.2)

project(EVPath VERSION 4.4.0 LANGUAGES C)

project(EVPath VERSION 4.4.0 LANGUAGES C CXX)



if(POLICY CMP0074)

  cmake_policy(SET CMP0074 NEW)
@@ -148,7 +148,7 @@ set(_pkg_config_private_libs) 
include(CheckCSourceCompiles)

set(LIBM_TEST_SOURCE "#include<math.h>\nfloat f; int main(){sqrt(f);return 0;}")

check_c_source_compiles("${LIBM_TEST_SOURCE}" HAVE_MATH)

if(NOT HAVE_SQRT)

if(NOT HAVE_MATH)

  set(CMAKE_REQUIRED_LIBRARIES m)

  check_c_source_compiles("${LIBM_TEST_SOURCE}" HAVE_LIBM_MATH)

  unset(CMAKE_REQUIRED_LIBRARIES)
@@ -172,6 +172,16 @@ if(NOT DEFINED CM_SELF_FORMATS) 
  set(CM_SELF_FORMATS 1)

endif()



if (DEFINED EVPATH_NO_RDMA) 

  option(EVPATH_NO_RDMA "Don't use any RDMA library"

    "${EVPATH_NO_RDMA}")

  if (EVPATH_NO_RDMA)

    SET (EVPATH_USE_LIBFABRIC OFF)

    SET (EVPATH_USE_NNTI OFF)

    SET (EVPATH_USE_IBVERBS OFF)

  endif()

endif()



cmake_dependent_option(EVPATH_TRANSPORT_MODULES

  "Enable stand-alone transport modules" ${BUILD_SHARED_LIBS}

  "SHARED_LIBS_SUPPORTED" OFF
@@ -196,7 +206,16 @@ if(Threads_FOUND) 
  list(APPEND _pkg_config_private_libs ${CMAKE_THREAD_LIBS_INIT})

endif()



if(DEFINED EVPATH_USE_NVML)

  option(EVPATH_USE_NVML "Use the NVidia management library"

    "${EVPATH_USE_NVML}")

  if(EVPATH_USE_NVML)

    find_package(NVML REQUIRED)

  endif()

else()

  find_package(NVML)

  option(EVPATH_USE_NVML "Use the NVidia management library" ${NVML_FOUND})

endif()

if(NVML_FOUND)

  if(NOT BUILD_SHARED_LIBS)

    list(APPEND EXTRA_INSTALL_PACKAGES NVML)
@@ -252,7 +271,15 @@ set(RUN_NNTI_TESTS FALSE) 


# enet transport

set(RUN_ENET_TESTS FALSE)

if(DEFINED EVPATH_USE_ENET)

  option(EVPATH_USE_ENET "Build the enet transport" "${EVPATH_USE_ENET}")

  if(EVPATH_USE_ENET)

    find_package(enet 1.3.13 REQUIRED)

  endif()

else()

  find_package(enet 1.3.13)

  option(EVPATH_USE_ENET "Build the enet transport" ${ENET_FOUND})

endif()

if(ENET_FOUND)

  set(RUN_ENET_TESTS TRUE)

  if(EVPATH_TRANSPORT_MODULES)
@@ -273,8 +300,49 @@ else() 
  message(STATUS " - Enet library was not found.  This is not a fatal error, just that the Enet transport will not be built.")

endif()



# udt4 transport

set(RUN_UDT4_TESTS FALSE)

if(DEFINED EVPATH_USE_UDT4)

  option(EVPATH_USE_UDT4 "Build the udt4 transport" "${EVPATH_USE_UDT4}")

  if(EVPATH_USE_UDT4)

    find_package(udt4 4.11 REQUIRED)

  endif()

else()

  find_package(udt4 4.11)

  option(EVPATH_USE_UDT4 "Build the udt4 transport" ${UDT4_FOUND})

endif()

if(UDT4_FOUND)

  set(RUN_UDT4_TESTS FALSE)   # DON'T TEST.  UDT4 NOT USEFUL

  if(EVPATH_TRANSPORT_MODULES)

    add_library(cmudt4 MODULE cmudt4.cpp ip_config.c)

    set_target_properties(cmudt4 PROPERTIES

      OUTPUT_NAME ${EVPATH_LIBRARY_PREFIX}cmudt4)



    target_link_libraries(cmudt4 evpath udt4::udt4)

    list(APPEND EVPATH_TRANSPORT_TARGETS cmudt4)

  else()

    target_sources(EVPath PRIVATE cmudt4.cpp)

    target_link_libraries(EVPath PRIVATE udt4::udt4)

    _pkg_get_target_prefix(udt4::udt4 udt4_PREFIX)

    list(APPEND _pkg_config_pfxs "${udt4_PREFIX}")

    list(APPEND _pkg_config_private_reqs "udt4 >= ${udt4_VERSION}")

  endif()

else()

  message(STATUS " - Udt4 library was not found.  This is not a fatal error, just that the Udt4 transport will not be built.")

endif()



# libfabric transport

if(DEFINED EVPATH_USE_LIBFABRIC)

  option(EVPATH_USE_LIBFABRIC "Build the libfabric transport"

    "${EVPATH_USE_LIBFABRIC}")

  if(EVPATH_USE_LIBFABRIC)

    find_package(LIBFABRIC REQUIRED)

  endif()

else()

  find_package(LIBFABRIC)

  option(EVPATH_USE_LIBFABRIC "Build the libfabric transport"

    ${LIBFABRIC_FOUND})

endif()

if(LIBFABRIC_FOUND) 

  if(EVPATH_TRANSPORT_MODULES)

    add_library(cmfabric MODULE cmfabric.c ip_config.c)
@@ -292,13 +360,25 @@ if(LIBFABRIC_FOUND) 
    list(APPEND _pkg_config_private_reqs "libfabric >= ${LIBFABRIC_VERSION}")

  endif()

else()

  if (EVPATH_USE_LIBFABRIC)

    message(STATUS " - LibFabric package was not found.  This is not a fatal error, just that the fabric transport will not be built.")

  endif()

endif()



# ibverbs transport

set(RUN_IB_TESTS FALSE)

CHECK_LIBRARY_EXISTS(ibverbs ibv_create_qp "" HAVE_IBVERBS)

if(HAVE_IBVERBS)

if(DEFINED EVPATH_USE_IBVERBS)

  option(EVPATH_USE_IBVERBS "Build the libfabric transport"

    "${EVPATH_USE_IBVERBS}")

  if(EVPATH_USE_IBVERBS)

    find_package(IBVERBS REQUIRED)

  endif()

else()

  find_package(IBVERBS)

  option(EVPATH_USE_IBVERBS "Build the libfabric transport" ${IBVERBS_FOUND})

endif()

set(HAVE_IBVERBS ${IBVERBS_FOUND})

if(IBVERBS_FOUND)

  if(BUILD_TESTING)

    if(NOT CMAKE_CROSSCOMPILING)

      message(STATUS "Check MEMLOCK rlimit for IB tests")
@@ -324,12 +404,12 @@ if(HAVE_IBVERBS) 
    add_library(cmib MODULE cmib.c)

    set_target_properties(cmib PROPERTIES

      OUTPUT_NAME ${EVPATH_LIBRARY_PREFIX}cmib)

    target_link_libraries(cmib evpath ibverbs)

    target_link_libraries(cmib evpath ${IBVERBS_LIBRARIES})

    list(APPEND EVPATH_TRANSPORT_TARGETS cmib)

  else()

    target_sources(EVPath PRIVATE cmib.c)

    target_link_libraries(EVPath PRIVATE ibverbs)

    list(APPEND _pkg_config_private_libs ibverbs)

    target_link_libraries(EVPath PRIVATE ${IBVERBS_LIBRARIES})

    list(APPEND _pkg_config_private_libs ${IBVERBS_LIBRARIES})

  endif()

  set(IB_FOUND 1)

endif()
@@ -344,10 +424,16 @@ if(NOT (DEFINED CercsArch)) 
  mark_as_advanced(CercsArch)

endif()



CHECK_LIBRARY_EXISTS(ugni GNI_CqCreate "" HAVE_UGNI)



if(HAVE_IBVERBS OR HAVE_UGNI)

if(IBVERBS_FOUND)

  if(DEFINED EVPATH_USE_NNTI)

    option(EVPATH_USE_NNTI "Build with nnti support" "${EVPATH_USE_NNTI}")

    if(EVPATH_USE_NNTI)

      find_package(NNTI REQUIRED)

    endif()

  else()

    find_package(NNTI)

    option(EVPATH_USE_NNTI "Build the nnti transport" ${NNTI_FOUND})

  endif()

  if(NNTI_FOUND)

    if(CercsArch STREQUAL "ppc64")

      set(RUN_NNTI_TESTS FALSE)
@@ -367,6 +453,8 @@ if(HAVE_IBVERBS OR HAVE_UGNI) 
      list(APPEND _pkg_config_private_libs ${NNTI_LIBRARIES})

    endif()

  endif()

else()

  set(EVPATH_USE_NNTI FALSE "Build the nnti transport" FORCE)

endif()



# Extra find module dependencies
@@ -591,7 +679,14 @@ if(NOT EVPATH_QUIET) 
  message(STATUS "CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")

  message(STATUS "BUILD_TESTING = ${BUILD_TESTING}  (options are: ON, OFF)")

  message(STATUS "BUILD_SHARED_LIBS = ${BUILD_SHARED_LIBS}  (options are: ON, OFF, default OFF)")

  message(STATUS "EVPATH_USE_NVML = ${EVPATH_USE_NVML}")

  message(STATUS "EVPATH_USE_ENET = ${EVPATH_USE_ENET}")

  message(STATUS "EVPATH_USE_UDT4 = ${EVPATH_USE_UDT4}")

  message(STATUS "EVPATH_USE_LIBFABRIC = ${EVPATH_USE_LIBFABRIC}")

  message(STATUS "EVPATH_USE_IBVERBS = ${EVPATH_USE_IBVERBS}")

  message(STATUS "EVPATH_USE_NNTI = ${EVPATH_USE_NNTI}")

  message(STATUS "EVPATH_TRANSPORT_MODULES = ${EVPATH_TRANSPORT_MODULES} (${EVPATH_TRANSPORT_TARGETS})")

  message(STATUS "Change a value with: cmake -D<Variable>=<Value>" )

  message(STATUS "To turn off all RDMA transports (fabric,ib,nnti) in EVPATH set EVPATH_NO_RDMA=1" )

  message(STATUS "-----------------------------------------------------------------------------" )

endif()

cm.c

+39 −13
Original line number Diff line number Diff line
@@ -74,9 +74,9 @@ void wait_for_pending_write(CMConnection conn); 
static void cm_wake_any_pending_write(CMConnection conn);

static void transport_wake_any_pending_write(CMConnection conn);

static void cm_set_pending_write(CMConnection conn);

static int drop_CM_lock(CManager cm, char *file, int line);

static int acquire_CM_lock(CManager cm, char *file, int line);

static int return_CM_lock_status(CManager cm, char *file, int line);

static int drop_CM_lock(CManager cm, const char *file, int line);

static int acquire_CM_lock(CManager cm, const char *file, int line);

static int return_CM_lock_status(CManager cm, const char *file, int line);

static void add_buffer_to_pending_queue(CManager cm, CMConnection conn, CMbuffer buf, long length);

static void cond_wait_CM_lock(CManager cm, void *cond, char *file, int line);
@@ -130,20 +130,20 @@ static void cond_wait_CM_lock(CManager cm, void *vcond, char *file, int line) 
    cm->locked++;

}



static int drop_CM_lock(CManager cm, char *file, int line)

static int drop_CM_lock(CManager cm, const char *file, int line)

{

    int ret = cm->locked;

    IntCManager_unlock(cm, file, line);

    return ret;

}



static int acquire_CM_lock(CManager cm, char *file, int line)

static int acquire_CM_lock(CManager cm, const char *file, int line)

{

    IntCManager_lock(cm, file, line);

    return cm->locked;

}



static int return_CM_lock_status(CManager cm, char *file, int line)

static int return_CM_lock_status(CManager cm, const char *file, int line)

{

    (void) file;

    (void) line;
@@ -2214,6 +2214,18 @@ INT_CMConnection_failed(CMConnection conn) 
	   }

 #endif

	   CManager_unlock(cm);

	   switch (*(int*)buffer) {

	   case 0x5042494f:

	   case 0x4f494250:  /* incoming FFS format protocol message */

	     {

	       extern int CM_pbio_query(CMConnection conn, CMTransport trans,

					char *buffer, long length);

	       

	       int ret = CM_pbio_query(conn, conn->trans, buffer, length);

	       CManager_lock(cm);

	       return ret;

	     }

	   }

	   ret = CMdo_non_CM_handler(conn, *(int*)buffer, buffer, length);

	   CManager_lock(cm);

	   if (local) cm_return_data_buf(cm, local);
@@ -3543,6 +3555,9 @@ INT_CMConnection_failed(CMConnection conn) 
     SelectInitFunc select_free_function = (SelectInitFunc)task_data[0];

     CMtrace_out(cm, CMFreeVerbose, "calling select FREE function, %p\n", task_data[1]);

     select_free_function(&CMstatic_trans_svcs, cm, &task_data[1]);

#if !NO_DYNAMIC_LINKING

     CMdlclose(task_data[2]);

#endif

     free(task_data);

 }
@@ -3554,6 +3569,7 @@ INT_CMConnection_failed(CMConnection conn) 
     SelectInitFunc init_function;

     SelectInitFunc shutdown_function;

     SelectInitFunc select_free_function;

     void *dlhandle = NULL;

 #if !NO_DYNAMIC_LINKING

     char *libname;

     lt_dlhandle handle;
@@ -3563,6 +3579,7 @@ INT_CMConnection_failed(CMConnection conn) 
     strcpy(libname, "lib" CM_LIBRARY_PREFIX "cmselect");

     strcat(libname, MODULE_EXT);

     handle = CMdlopen(cm->CMTrace_file, libname, 0);

     dlhandle = handle;

     free(libname);

     if (!handle) {

	 fprintf(stderr, "Failed to load required select dll.\n");
@@ -3616,9 +3633,10 @@ INT_CMConnection_failed(CMConnection conn) 
     CMtrace_out(cm, CMFreeVerbose, "CManager adding select shutdown function, %lx\n",(long)shutdown_function);

     internal_add_shutdown_task(cm, select_shutdown, (void*)shutdown_function, SHUTDOWN_TASK);

     {

	 void ** data = malloc(2 * sizeof(void*));

	 void ** data = malloc(3 * sizeof(void*));

	 data[0] = select_free_function;

	 data[1] = cm->control_list->select_data;

	 data[2] = dlhandle;

	 internal_add_shutdown_task(cm, select_free, (void*)data, FREE_TASK);

     }

 }
@@ -3659,6 +3677,13 @@ INT_CMConnection_failed(CMConnection conn) 
static handler_list foreign_handler_list;

static int foreign_handler_count = 0;



static void

clear_foreign_handlers()

{

    if (foreign_handler_count == 0) return;

    free(foreign_handler_list);

}



 extern void

 INT_CMregister_non_CM_message_handler(int header, CMNonCMHandler handler)

 {
@@ -3668,6 +3693,7 @@ INT_CMConnection_failed(CMConnection conn) 
					  (foreign_handler_count + 1));

     } else {

	 foreign_handler_list = INT_CMmalloc(sizeof(foreign_handler_list[0]));

	 atexit(clear_foreign_handlers);

     }

     foreign_handler_list[foreign_handler_count].header = header;

     foreign_handler_list[foreign_handler_count].handler = handler;

cm_internal.h

+3 −3
Original line number Diff line number Diff line
@@ -300,8 +300,8 @@ struct _CMFormat { 


#define CManager_lock(cm) IntCManager_lock(cm, __FILE__, __LINE__)

#define CManager_unlock(cm) IntCManager_unlock(cm, __FILE__, __LINE__)

extern void IntCManager_lock(CManager cm, char *file, int line);

extern void IntCManager_unlock(CManager cm, char *file, int line);

extern void IntCManager_lock(CManager cm, const char *file, int line);

extern void IntCManager_unlock(CManager cm, const char *file, int line);

extern int CManager_locked(CManager cm);

extern void CMControlList_lock(CMControlList cl);

extern void CMControlList_unlock(CMControlList cl);
@@ -335,7 +335,7 @@ CMWriteQueuedData(transport_entry trans, CMConnection conn); 
extern CMincoming_format_list

CMidentify_CMformat(CManager cm, FFSTypeHandle format);



extern void CMtransport_trace(CManager cm, char *format, ...);

extern void CMtransport_trace(CManager cm, const char *format, ...);



extern void

CM_fd_add_select(CManager cm, int fd, select_list_func handler_func,