Commit 8e70a7c5 authored by Gunther Weber's avatar Gunther Weber Committed by Kitware Robot
Browse files

Merge topic 'fix-distributed-contour-tree-hang'

e4a46f33 Removed two TODO comments after verifying parameters
3ef95ded Port bug fix from distributed to augmented contour tree filter
70f7337f

 Fix hang in distributed contour tree

Acked-by: default avatarKitware Robot <kwrobot@kitware.com>
Acked-by: default avatarKenneth Moreland <kmorel@sandia.gov>
Merge-request: !2424
parents f1cb6d83 e4a46f33
......@@ -337,25 +337,33 @@ VTKM_CONT void ContourTreeAugmented::DoPostExecute(
);
// Compute the gids for our local blocks
using RegularDecomposer = vtkmdiy::RegularDecomposer<vtkmdiy::DiscreteBounds>;
const vtkm::worklet::contourtree_distributed::SpatialDecomposition& spatialDecomp =
this->MultiBlockTreeHelper->MultiBlockSpatialDecomposition;
const auto numDims = spatialDecomp.NumberOfDimensions();
// ... division vector
RegularDecomposer::DivisionsVector diyDivisions(numDims);
for (vtkm::IdComponent d = 0;
d < static_cast<vtkm::IdComponent>(spatialDecomp.NumberOfDimensions());
++d)
{
diyDivisions[d] = static_cast<int>(spatialDecomp.BlocksPerDimension[d]);
}
// ... coordinates of local blocks
auto localBlockIndicesPortal = spatialDecomp.LocalBlockIndices.ReadPortal();
std::vector<vtkm::Id> vtkmdiyLocalBlockGids(static_cast<size_t>(input.GetNumberOfPartitions()));
for (vtkm::Id bi = 0; bi < input.GetNumberOfPartitions(); bi++)
{
std::vector<int> tempCoords; // DivisionsVector type in DIY
std::vector<int> tempDivisions; // DivisionsVector type in DIY
tempCoords.resize(static_cast<size_t>(spatialDecomp.NumberOfDimensions()));
tempDivisions.resize(static_cast<size_t>(spatialDecomp.NumberOfDimensions()));
RegularDecomposer::DivisionsVector diyCoords(static_cast<size_t>(numDims));
auto currentCoords = localBlockIndicesPortal.Get(bi);
for (std::size_t di = 0; di < static_cast<size_t>(spatialDecomp.NumberOfDimensions()); di++)
for (vtkm::IdComponent d = 0; d < numDims; ++d)
{
tempCoords[di] = static_cast<int>(currentCoords[static_cast<vtkm::IdComponent>(di)]);
tempDivisions[di] =
static_cast<int>(spatialDecomp.BlocksPerDimension[static_cast<vtkm::IdComponent>(di)]);
diyCoords[d] = static_cast<int>(currentCoords[d]);
}
vtkmdiyLocalBlockGids[static_cast<size_t>(bi)] =
vtkmdiy::RegularDecomposer<vtkmdiy::DiscreteBounds>::coords_to_gid(tempCoords, tempDivisions);
RegularDecomposer::coords_to_gid(diyCoords, diyDivisions);
}
// Add my local blocks to the vtkmdiy master.
......@@ -367,10 +375,16 @@ VTKM_CONT void ContourTreeAugmented::DoPostExecute(
}
// Define the decomposition of the domain into regular blocks
vtkmdiy::RegularDecomposer<vtkmdiy::DiscreteBounds> decomposer(
static_cast<int>(spatialDecomp.NumberOfDimensions()), // number of dims
RegularDecomposer::BoolVector shareFace(3, true);
RegularDecomposer::BoolVector wrap(3, false);
RegularDecomposer::CoordinateVector ghosts(3, 1);
RegularDecomposer decomposer(static_cast<int>(numDims),
spatialDecomp.GetVTKmDIYBounds(),
static_cast<int>(spatialDecomp.GetGlobalNumberOfBlocks()));
static_cast<int>(spatialDecomp.GetGlobalNumberOfBlocks()),
shareFace,
wrap,
ghosts,
diyDivisions);
// Define which blocks live on which rank so that vtkmdiy can manage them
vtkmdiy::DynamicAssigner assigner(
......@@ -381,14 +395,14 @@ VTKM_CONT void ContourTreeAugmented::DoPostExecute(
static_cast<int>(vtkmdiyLocalBlockGids[static_cast<size_t>(bi)]));
}
// Fix the vtkmdiy links. TODO Remove changes to the vtkmdiy in VTKM when we update to the new VTK-M
// Fix the vtkmdiy links. (NOTE: includes an MPI barrier)
vtkmdiy::fix_links(master, assigner);
// partners for merge over regular block grid
vtkmdiy::RegularMergePartners partners(
decomposer, // domain decomposition
2, // raix of k-ary reduction. TODO check this value
true // contiguous: true=distance doubling , false=distnace halving TODO check this value
2, // raix of k-ary reduction.
true // contiguous: true=distance doubling , false=distnace halving
);
// reduction
vtkmdiy::reduce(
......
......@@ -741,30 +741,35 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute(
timer.Start();
// 1.2.1 Compute the gids for our local blocks
using RegularDecomposer = vtkmdiy::RegularDecomposer<vtkmdiy::DiscreteBounds>;
const vtkm::worklet::contourtree_distributed::SpatialDecomposition& spatialDecomp =
this->MultiBlockSpatialDecomposition;
const auto numDims = spatialDecomp.NumberOfDimensions();
// ... division vector
RegularDecomposer::DivisionsVector diyDivisions(numDims);
for (vtkm::IdComponent d = 0;
d < static_cast<vtkm::IdComponent>(spatialDecomp.NumberOfDimensions());
++d)
{
diyDivisions[d] = static_cast<int>(spatialDecomp.BlocksPerDimension[d]);
}
// ... coordinates of local blocks
auto localBlockIndicesPortal = spatialDecomp.LocalBlockIndices.ReadPortal();
std::vector<vtkm::Id> vtkmdiyLocalBlockGids(static_cast<size_t>(input.GetNumberOfPartitions()));
for (vtkm::Id bi = 0; bi < input.GetNumberOfPartitions(); bi++)
{
std::vector<int> tempCoords; // DivisionsVector type in DIY
std::vector<int> tempDivisions; // DivisionsVector type in DIY
tempCoords.resize(static_cast<size_t>(spatialDecomp.NumberOfDimensions()));
tempDivisions.resize(static_cast<size_t>(spatialDecomp.NumberOfDimensions()));
RegularDecomposer::DivisionsVector diyCoords(static_cast<size_t>(numDims));
auto currentCoords = localBlockIndicesPortal.Get(bi);
for (std::size_t di = 0; di < static_cast<size_t>(spatialDecomp.NumberOfDimensions()); di++)
for (vtkm::IdComponent d = 0; d < numDims; ++d)
{
tempCoords[di] = static_cast<int>(currentCoords[static_cast<vtkm::IdComponent>(di)]);
tempDivisions[di] =
static_cast<int>(spatialDecomp.BlocksPerDimension[static_cast<vtkm::IdComponent>(di)]);
diyCoords[d] = static_cast<int>(currentCoords[d]);
}
vtkmdiyLocalBlockGids[static_cast<size_t>(bi)] =
vtkmdiy::RegularDecomposer<vtkmdiy::DiscreteBounds>::coords_to_gid(tempCoords, tempDivisions);
RegularDecomposer::coords_to_gid(diyCoords, diyDivisions);
}
std::vector<vtkmdiy::Link*> localLinks(static_cast<std::vector<vtkmdiy::Link>::size_type>(
input.GetNumberOfPartitions())); // dummy links needed to make DIY happy
// Record time to compute the local block ids
timingsStream << " " << std::setw(38) << std::left << "Compute Block Ids and Local Links"
<< ": " << timer.GetElapsedTime() << " seconds" << std::endl;
......@@ -773,10 +778,13 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute(
// 1.2.2 Add my local blocks to the vtkmdiy master.
for (std::size_t bi = 0; bi < static_cast<std::size_t>(input.GetNumberOfPartitions()); bi++)
{
localLinks[bi] = new vtkmdiy::Link;
master.add(static_cast<int>(vtkmdiyLocalBlockGids[bi]), // block id
localDataBlocks[bi],
localLinks[bi]);
new vtkmdiy::Link); // Use dummy link to make DIY happy.
// NOTE: The dummy link is never used, since all communication is via RegularDecomposer,
// which sets up its own links
// NOTE: No need to keep the pointer, as DIY will "own" it and delete it when no longer
// needed TODO/FIXME: Confirm that last statement
}
// Record time for dding data blocks to the master
......@@ -785,10 +793,16 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute(
timer.Start();
// 1.2.3 Define the decomposition of the domain into regular blocks
vtkmdiy::RegularDecomposer<vtkmdiy::DiscreteBounds> decomposer(
static_cast<int>(spatialDecomp.NumberOfDimensions()), // number of dims
RegularDecomposer::BoolVector shareFace(3, true);
RegularDecomposer::BoolVector wrap(3, false);
RegularDecomposer::CoordinateVector ghosts(3, 1);
RegularDecomposer decomposer(static_cast<int>(numDims),
spatialDecomp.GetVTKmDIYBounds(),
static_cast<int>(spatialDecomp.GetGlobalNumberOfBlocks()));
static_cast<int>(spatialDecomp.GetGlobalNumberOfBlocks()),
shareFace,
wrap,
ghosts,
diyDivisions);
// Define which blocks live on which rank so that vtkmdiy can manage them
vtkmdiy::DynamicAssigner assigner(
......@@ -815,8 +829,8 @@ VTKM_CONT void ContourTreeUniformDistributed::DoPostExecute(
// partners for merge over regular block grid
vtkmdiy::RegularSwapPartners partners(
decomposer, // domain decomposition
2, // raix of k-ary reduction. TODO check this value
true // contiguous: true=distance doubling , false=distnace halving TODO check this value
2, // radix of k-ary reduction.
true // contiguous: true=distance doubling, false=distance halving
);
// Record time to create the swap partners
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment