Commit 8e70a7c5 authored by Gunther Weber's avatar Gunther Weber Committed by Kitware Robot
Browse files

Merge topic 'fix-distributed-contour-tree-hang'

e4a46f33 Removed two TODO comments after verifying parameters
3ef95ded Port bug fix from distributed to augmented contour tree filter
70f7337f

 Fix hang in distributed contour tree

Acked-by: default avatarKitware Robot <kwrobot@kitware.com>
Acked-by: default avatarKenneth Moreland <kmorel@sandia.gov>
Merge-request: !2424
parents f1cb6d83 e4a46f33
...@@ -337,25 +337,33 @@ VTKM_CONT void ContourTreeAugmented::DoPostExecute( ...@@ -337,25 +337,33 @@ VTKM_CONT void ContourTreeAugmented::DoPostExecute(
); );
// Compute the gids for our local blocks // Compute the gids for our local blocks
using RegularDecomposer = vtkmdiy::RegularDecomposer<vtkmdiy::DiscreteBounds>;
const vtkm::worklet::contourtree_distributed::SpatialDecomposition& spatialDecomp = const vtkm::worklet::contourtree_distributed::SpatialDecomposition& spatialDecomp =
this->MultiBlockTreeHelper->MultiBlockSpatialDecomposition; this->MultiBlockTreeHelper->MultiBlockSpatialDecomposition;
const auto numDims = spatialDecomp.NumberOfDimensions();
// ... division vector
RegularDecomposer::DivisionsVector diyDivisions(numDims);
for (vtkm::IdComponent d = 0;
d < static_cast<vtkm::IdComponent>(spatialDecomp.NumberOfDimensions());
++d)
{
diyDivisions[d] = static_cast<int>(spatialDecomp.BlocksPerDimension[d]);
}
// ... coordinates of local blocks
auto localBlockIndicesPortal = spatialDecomp.LocalBlockIndices.ReadPortal(); auto localBlockIndicesPortal = spatialDecomp.LocalBlockIndices.ReadPortal();
std::vector<vtkm::Id> vtkmdiyLocalBlockGids(static_cast<size_t>(input.GetNumberOfPartitions())); std::vector<vtkm::Id> vtkmdiyLocalBlockGids(static_cast<size_t>(input.GetNumberOfPartitions()));
for (vtkm::Id bi = 0; bi < input.GetNumberOfPartitions(); bi++) for (vtkm::Id bi = 0; bi < input.GetNumberOfPartitions(); bi++)
{ {
std::vector<int> tempCoords; // DivisionsVector type in DIY RegularDecomposer::DivisionsVector diyCoords(static_cast<size_t>(numDims));
std::vector<int> tempDivisions; // DivisionsVector type in DIY
tempCoords.resize(static_cast<size_t>(spatialDecomp.NumberOfDimensions()));
tempDivisions.resize(static_cast<size_t>(spatialDecomp.NumberOfDimensions()));
auto currentCoords = localBlockIndicesPortal.Get(bi); auto currentCoords = localBlockIndicesPortal.Get(bi);
for (std::size_t di = 0; di < static_cast<size_t>(spatialDecomp.NumberOfDimensions()); di++) for (vtkm::IdComponent d = 0; d < numDims; ++d)
{ {
tempCoords[di] = static_cast<int>(currentCoords[static_cast<vtkm::IdComponent>(di)]); diyCoords[d] = static_cast<int>(currentCoords[d]);
tempDivisions[di] =
static_cast<int>(spatialDecomp.BlocksPerDimension[static_cast<vtkm::IdComponent>(di)]);
} }
vtkmdiyLocalBlockGids[static_cast<size_t>(bi)] = vtkmdiyLocalBlockGids[static_cast<size_t>(bi)] =
vtkmdiy::RegularDecomposer<vtkmdiy::DiscreteBounds>::coords_to_gid(tempCoords, tempDivisions); RegularDecomposer::coords_to_gid(diyCoords, diyDivisions);
} }
// Add my local blocks to the vtkmdiy master. // Add my local blocks to the vtkmdiy master.
...@@ -367,10 +375,16 @@ VTKM_CONT void ContourTreeAugmented::DoPostExecute( ...@@ -367,10 +375,16 @@ VTKM_CONT void ContourTreeAugmented::DoPostExecute(
} }
// Define the decomposition of the domain into regular blocks // Define the decomposition of the domain into regular blocks
vtkmdiy::RegularDecomposer<vtkmdiy::DiscreteBounds> decomposer( RegularDecomposer::BoolVector shareFace(3, true);
static_cast<int>(spatialDecomp.NumberOfDimensions()), // number of dims RegularDecomposer::BoolVector wrap(3, false);
spatialDecomp.GetVTKmDIYBounds(), RegularDecomposer::CoordinateVector ghosts(3, 1);
static_cast<int>(spatialDecomp.GetGlobalNumberOfBlocks())); RegularDecomposer decomposer(static_cast<int>(numDims),
spatialDecomp.GetVTKmDIYBounds(),
static_cast<int>(spatialDecomp.GetGlobalNumberOfBlocks()),
shareFace,
wrap,
ghosts,
diyDivisions);
// Define which blocks live on which rank so that vtkmdiy can manage them // Define which blocks live on which rank so that vtkmdiy can manage them
vtkmdiy::DynamicAssigner assigner( vtkmdiy::DynamicAssigner assigner(
...@@ -381,14 +395,14 @@ VTKM_CONT void ContourTreeAugmented::DoPostExecute( ...@@ -381,14 +395,14 @@ VTKM_CONT void ContourTreeAugmented::DoPostExecute(
static_cast<int>(vtkmdiyLocalBlockGids[static_cast<size_t>(bi)])); static_cast<int>(vtkmdiyLocalBlockGids[static_cast<size_t>(bi)]));
} }
// Fix the vtkmdiy links. TODO Remove changes to the vtkmdiy in VTKM when we update to the new VTK-M // Fix the vtkmdiy links. (NOTE: includes an MPI barrier)
vtkmdiy::fix_links(master, assigner); vtkmdiy::fix_links(master, assigner);
// partners for merge over regular block grid // partners for merge over regular block grid
vtkmdiy::RegularMergePartners partners( vtkmdiy::RegularMergePartners partners(
decomposer, // domain decomposition decomposer, // domain decomposition
2, // raix of k-ary reduction. TODO check this value 2, // raix of k-ary reduction.
true // contiguous: true=distance doubling , false=distnace halving TODO check this value true // contiguous: true=distance doubling , false=distnace halving
); );
// reduction // reduction
vtkmdiy::reduce( vtkmdiy::reduce(
......
...@@ -741,30 +741,35 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute( ...@@ -741,30 +741,35 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute(
timer.Start(); timer.Start();
// 1.2.1 Compute the gids for our local blocks // 1.2.1 Compute the gids for our local blocks
using RegularDecomposer = vtkmdiy::RegularDecomposer<vtkmdiy::DiscreteBounds>;
const vtkm::worklet::contourtree_distributed::SpatialDecomposition& spatialDecomp = const vtkm::worklet::contourtree_distributed::SpatialDecomposition& spatialDecomp =
this->MultiBlockSpatialDecomposition; this->MultiBlockSpatialDecomposition;
const auto numDims = spatialDecomp.NumberOfDimensions();
// ... division vector
RegularDecomposer::DivisionsVector diyDivisions(numDims);
for (vtkm::IdComponent d = 0;
d < static_cast<vtkm::IdComponent>(spatialDecomp.NumberOfDimensions());
++d)
{
diyDivisions[d] = static_cast<int>(spatialDecomp.BlocksPerDimension[d]);
}
// ... coordinates of local blocks
auto localBlockIndicesPortal = spatialDecomp.LocalBlockIndices.ReadPortal(); auto localBlockIndicesPortal = spatialDecomp.LocalBlockIndices.ReadPortal();
std::vector<vtkm::Id> vtkmdiyLocalBlockGids(static_cast<size_t>(input.GetNumberOfPartitions())); std::vector<vtkm::Id> vtkmdiyLocalBlockGids(static_cast<size_t>(input.GetNumberOfPartitions()));
for (vtkm::Id bi = 0; bi < input.GetNumberOfPartitions(); bi++) for (vtkm::Id bi = 0; bi < input.GetNumberOfPartitions(); bi++)
{ {
std::vector<int> tempCoords; // DivisionsVector type in DIY RegularDecomposer::DivisionsVector diyCoords(static_cast<size_t>(numDims));
std::vector<int> tempDivisions; // DivisionsVector type in DIY
tempCoords.resize(static_cast<size_t>(spatialDecomp.NumberOfDimensions()));
tempDivisions.resize(static_cast<size_t>(spatialDecomp.NumberOfDimensions()));
auto currentCoords = localBlockIndicesPortal.Get(bi); auto currentCoords = localBlockIndicesPortal.Get(bi);
for (std::size_t di = 0; di < static_cast<size_t>(spatialDecomp.NumberOfDimensions()); di++) for (vtkm::IdComponent d = 0; d < numDims; ++d)
{ {
tempCoords[di] = static_cast<int>(currentCoords[static_cast<vtkm::IdComponent>(di)]); diyCoords[d] = static_cast<int>(currentCoords[d]);
tempDivisions[di] =
static_cast<int>(spatialDecomp.BlocksPerDimension[static_cast<vtkm::IdComponent>(di)]);
} }
vtkmdiyLocalBlockGids[static_cast<size_t>(bi)] = vtkmdiyLocalBlockGids[static_cast<size_t>(bi)] =
vtkmdiy::RegularDecomposer<vtkmdiy::DiscreteBounds>::coords_to_gid(tempCoords, tempDivisions); RegularDecomposer::coords_to_gid(diyCoords, diyDivisions);
} }
std::vector<vtkmdiy::Link*> localLinks(static_cast<std::vector<vtkmdiy::Link>::size_type>(
input.GetNumberOfPartitions())); // dummy links needed to make DIY happy
// Record time to compute the local block ids // Record time to compute the local block ids
timingsStream << " " << std::setw(38) << std::left << "Compute Block Ids and Local Links" timingsStream << " " << std::setw(38) << std::left << "Compute Block Ids and Local Links"
<< ": " << timer.GetElapsedTime() << " seconds" << std::endl; << ": " << timer.GetElapsedTime() << " seconds" << std::endl;
...@@ -773,10 +778,13 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute( ...@@ -773,10 +778,13 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute(
// 1.2.2 Add my local blocks to the vtkmdiy master. // 1.2.2 Add my local blocks to the vtkmdiy master.
for (std::size_t bi = 0; bi < static_cast<std::size_t>(input.GetNumberOfPartitions()); bi++) for (std::size_t bi = 0; bi < static_cast<std::size_t>(input.GetNumberOfPartitions()); bi++)
{ {
localLinks[bi] = new vtkmdiy::Link;
master.add(static_cast<int>(vtkmdiyLocalBlockGids[bi]), // block id master.add(static_cast<int>(vtkmdiyLocalBlockGids[bi]), // block id
localDataBlocks[bi], localDataBlocks[bi],
localLinks[bi]); new vtkmdiy::Link); // Use dummy link to make DIY happy.
// NOTE: The dummy link is never used, since all communication is via RegularDecomposer,
// which sets up its own links
// NOTE: No need to keep the pointer, as DIY will "own" it and delete it when no longer
// needed TODO/FIXME: Confirm that last statement
} }
// Record time for dding data blocks to the master // Record time for dding data blocks to the master
...@@ -785,10 +793,16 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute( ...@@ -785,10 +793,16 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute(
timer.Start(); timer.Start();
// 1.2.3 Define the decomposition of the domain into regular blocks // 1.2.3 Define the decomposition of the domain into regular blocks
vtkmdiy::RegularDecomposer<vtkmdiy::DiscreteBounds> decomposer( RegularDecomposer::BoolVector shareFace(3, true);
static_cast<int>(spatialDecomp.NumberOfDimensions()), // number of dims RegularDecomposer::BoolVector wrap(3, false);
spatialDecomp.GetVTKmDIYBounds(), RegularDecomposer::CoordinateVector ghosts(3, 1);
static_cast<int>(spatialDecomp.GetGlobalNumberOfBlocks())); RegularDecomposer decomposer(static_cast<int>(numDims),
spatialDecomp.GetVTKmDIYBounds(),
static_cast<int>(spatialDecomp.GetGlobalNumberOfBlocks()),
shareFace,
wrap,
ghosts,
diyDivisions);
// Define which blocks live on which rank so that vtkmdiy can manage them // Define which blocks live on which rank so that vtkmdiy can manage them
vtkmdiy::DynamicAssigner assigner( vtkmdiy::DynamicAssigner assigner(
...@@ -815,8 +829,8 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute( ...@@ -815,8 +829,8 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute(
// partners for merge over regular block grid // partners for merge over regular block grid
vtkmdiy::RegularSwapPartners partners( vtkmdiy::RegularSwapPartners partners(
decomposer, // domain decomposition decomposer, // domain decomposition
2, // raix of k-ary reduction. TODO check this value 2, // radix of k-ary reduction.
true // contiguous: true=distance doubling , false=distnace halving TODO check this value true // contiguous: true=distance doubling, false=distance halving
); );
// Record time to create the swap partners // Record time to create the swap partners
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment