From 13be5b17e5a13e79f49c12f451a61de99fb4c00e Mon Sep 17 00:00:00 2001
From: Vickie Lynch <lynchve@ornl.gov>
Date: Wed, 12 Aug 2015 16:24:19 -0400
Subject: [PATCH] Refs #13350 gather mpi workspaces
---
Code/Mantid/Build/CMake/MPISetup.cmake | 2 +-
.../inc/MantidAPI/DataProcessorAlgorithm.h | 6 +++
.../API/src/DataProcessorAlgorithm.cpp | 40 +++++++++++++++++++
.../MPIAlgorithms/src/GatherWorkspaces.cpp | 10 ++---
.../src/LoadEventAndCompress.cpp | 13 ++----
5 files changed, 56 insertions(+), 15 deletions(-)
diff --git a/Code/Mantid/Build/CMake/MPISetup.cmake b/Code/Mantid/Build/CMake/MPISetup.cmake
index bcc8602360e..811bc6872b0 100644
--- a/Code/Mantid/Build/CMake/MPISetup.cmake
+++ b/Code/Mantid/Build/CMake/MPISetup.cmake
@@ -23,7 +23,7 @@ include_directories( ${Boost_INCLUDE_DIRS} )
# Add a definition that's used to guard MPI-specific parts of the main code
add_definitions ( -DMPI_BUILD )
-add_subdirectory( ${CMAKE_CURRENT_SOURCE_DIR}/Testing/SystemTests/scripts )
+#add_subdirectory( ${CMAKE_CURRENT_SOURCE_DIR}/Testing/SystemTests/scripts )
# Add the ability to build a 'mantid-mpi' rpm
set ( CPACK_PACKAGE_NAME mantid-mpi )
diff --git a/Code/Mantid/Framework/API/inc/MantidAPI/DataProcessorAlgorithm.h b/Code/Mantid/Framework/API/inc/MantidAPI/DataProcessorAlgorithm.h
index 143670ddf94..64ea4074d18 100644
--- a/Code/Mantid/Framework/API/inc/MantidAPI/DataProcessorAlgorithm.h
+++ b/Code/Mantid/Framework/API/inc/MantidAPI/DataProcessorAlgorithm.h
@@ -67,6 +67,7 @@ protected:
getProcessProperties(const std::string &propertyManager=std::string()) const;
/// MPI option. If false, we will use one job event if MPI is available
bool m_useMPI;
+ Workspace_sptr assemble(Workspace_sptr partialWS);
Workspace_sptr assemble(const std::string &partialWSName,
const std::string &outputWSName);
void saveNexus(const std::string &outputWSName,
@@ -91,10 +92,15 @@ protected:
/// Add a matrix workspace to another matrix workspace
MatrixWorkspace_sptr plus(const MatrixWorkspace_sptr lhs,
const MatrixWorkspace_sptr rhs);
+
/// Add a single value to a matrix workspace
MatrixWorkspace_sptr plus(const MatrixWorkspace_sptr lhs,
const double &rhsValue);
+ /// Add a matrix workspace to another matrix workspace
+ MatrixWorkspace_sptr plus2(MatrixWorkspace_sptr lhs,
+ MatrixWorkspace_sptr rhs);
+
/// Subract a matrix workspace by another matrix workspace
MatrixWorkspace_sptr minus(const MatrixWorkspace_sptr lhs,
const MatrixWorkspace_sptr rhs);
diff --git a/Code/Mantid/Framework/API/src/DataProcessorAlgorithm.cpp b/Code/Mantid/Framework/API/src/DataProcessorAlgorithm.cpp
index c28b9801800..5acdb37f0a5 100644
--- a/Code/Mantid/Framework/API/src/DataProcessorAlgorithm.cpp
+++ b/Code/Mantid/Framework/API/src/DataProcessorAlgorithm.cpp
@@ -196,6 +196,32 @@ MatrixWorkspace_sptr DataProcessorAlgorithm::loadChunk(const size_t rowIndex) {
"DataProcessorAlgorithm::loadChunk is not implemented");
}
+/**
+ * Assemble the partial workspaces from all MPI processes
+ * @param partialWS :: workspace to assemble
+ * thread only)
+ */
+Workspace_sptr
+DataProcessorAlgorithm::assemble(Workspace_sptr partialWS) {
+ Workspace_sptr outputWS =partialWS;
+#ifdef MPI_BUILD
+ IAlgorithm_sptr gatherAlg = createChildAlgorithm("GatherWorkspaces");
+ gatherAlg->setLogging(true);
+ gatherAlg->setAlwaysStoreInADS(true);
+ gatherAlg->setProperty("InputWorkspace", partialWS);
+ gatherAlg->setProperty("PreserveEvents", true);
+ gatherAlg->setPropertyValue("OutputWorkspace", "_total");
+ gatherAlg->execute();
+
+ if (isMainThread())
+ {
+ outputWS = AnalysisDataService::Instance().retrieve("_total");
+ }
+#endif
+
+ return outputWS;
+}
+
/**
* Assemble the partial workspaces from all MPI processes
* @param partialWSName :: Name of the workspace to assemble
@@ -451,6 +477,20 @@ DataProcessorAlgorithm::plus(const MatrixWorkspace_sptr lhs,
"Plus", lhs, rhs);
}
+/**
+ * Add a matrix workspace to another matrix workspace
+ * @param lhs :: the workspace on the left hand side of the addition symbol
+ * @param rhs :: the workspace on the right hand side of the addition symbol
+ * @return matrix workspace resulting from the operation
+ */
+MatrixWorkspace_sptr
+DataProcessorAlgorithm::plus2(MatrixWorkspace_sptr lhs,
+ MatrixWorkspace_sptr rhs) {
+ return this->executeBinaryAlgorithm<
+ MatrixWorkspace_sptr, MatrixWorkspace_sptr, MatrixWorkspace_sptr>(
+ "Plus", lhs, rhs);
+}
+
/**
* Add a single value to another matrix workspace
* @param lhs :: the workspace on the left hand side of the addition symbol
diff --git a/Code/Mantid/Framework/MPIAlgorithms/src/GatherWorkspaces.cpp b/Code/Mantid/Framework/MPIAlgorithms/src/GatherWorkspaces.cpp
index 20fbe05456c..abc1c580496 100644
--- a/Code/Mantid/Framework/MPIAlgorithms/src/GatherWorkspaces.cpp
+++ b/Code/Mantid/Framework/MPIAlgorithms/src/GatherWorkspaces.cpp
@@ -70,7 +70,7 @@ void GatherWorkspaces::init() {
// Output is optional - only the root process will output a workspace
declareProperty(new WorkspaceProperty<>(
"OutputWorkspace", "", Direction::Output, PropertyMode::Optional));
- declareProperty("PreserveEvents", false,
+ declareProperty("PreserveEvents", true,
"Keep the output workspace as an EventWorkspace, if the "
"input has events (default).\n"
"If false, then the workspace gets converted to a "
@@ -182,7 +182,7 @@ void GatherWorkspaces::exec() {
outputWorkspace->dataE(wi) = inputWorkspace->readE(wi);
const int numReqs(3 * (included.size() - 1));
- mpi::request reqs[numReqs];
+ std::vector<boost::mpi::request> reqs(numReqs);
int j(0);
// Receive data from all the other processes
@@ -201,7 +201,7 @@ void GatherWorkspaces::exec() {
}
// Make sure everything's been received before exiting the algorithm
- mpi::wait_all(reqs, reqs + numReqs);
+ mpi::wait_all(reqs.begin(), reqs.end());
}
ISpectrum *outSpec = outputWorkspace->getSpectrum(wi);
outSpec->clearDetectorIDs();
@@ -211,7 +211,7 @@ void GatherWorkspaces::exec() {
reduce(included, inputWorkspace->readY(wi), vplus(), 0);
reduce(included, inputWorkspace->readE(wi), eplus(), 0);
} else if (accum == "Append") {
- mpi::request reqs[3];
+ std::vector<boost::mpi::request> reqs(3);
// Send the spectrum to the root process
reqs[0] = included.isend(0, 0, inputWorkspace->readX(0));
@@ -219,7 +219,7 @@ void GatherWorkspaces::exec() {
reqs[2] = included.isend(0, 2, inputWorkspace->readE(0));
// Make sure the sends have completed before exiting the algorithm
- mpi::wait_all(reqs, reqs + 3);
+ mpi::wait_all(reqs.begin(), reqs.end());
}
}
}
diff --git a/Code/Mantid/Framework/WorkflowAlgorithms/src/LoadEventAndCompress.cpp b/Code/Mantid/Framework/WorkflowAlgorithms/src/LoadEventAndCompress.cpp
index c603ffdf9da..7ea14f5c70f 100644
--- a/Code/Mantid/Framework/WorkflowAlgorithms/src/LoadEventAndCompress.cpp
+++ b/Code/Mantid/Framework/WorkflowAlgorithms/src/LoadEventAndCompress.cpp
@@ -200,8 +200,7 @@ void LoadEventAndCompress::exec() {
m_chunkingTable = determineChunk(m_filename);
// first run is free
- EventWorkspace_sptr resultWS =
- boost::dynamic_pointer_cast<EventWorkspace>(loadChunk(0));
+ MatrixWorkspace_sptr resultWS = loadChunk(0);
processChunk(resultWS);
// load the other chunks
@@ -209,18 +208,14 @@ void LoadEventAndCompress::exec() {
for (size_t i = 1; i < numRows; ++i) {
MatrixWorkspace_sptr temp = loadChunk(i);
processChunk(temp);
- auto alg = createChildAlgorithm("Plus");
- alg->setProperty("LHSWorkspace", resultWS);
- alg->setProperty("RHSWorkspace", temp);
- alg->setProperty("OutputWorkspace", resultWS);
- alg->setProperty("ClearRHSWorkspace", true);
- alg->executeAsChildAlg();
+ resultWS = plus2(resultWS,temp);
}
+ Workspace_sptr total = assemble(resultWS);
// Don't bother compressing combined workspace. DetermineChunking is designed
// to prefer loading full banks so no further savings should be available.
- setProperty("OutputWorkspace", resultWS);
+ setProperty("OutputWorkspace", total);
}
} // namespace WorkflowAlgorithms
--
GitLab