Commit f85d6565 authored by Ethan Coon's avatar Ethan Coon
Browse files

updates data container to deal with N-Dimensional things, where N is 1 or...

updates data container to deal with N-Dimensional things, where N is 1 or more, and to partition in multiple dimensions.  This should make it easier if the time arrives to partition in multiple ways (i.e. to partition across PFTs if different PFTs get different physics)
parent 4f7171fd
......@@ -20,7 +20,7 @@
#include "legion.h"
#include "domains.hh"
#include "data.hh"
#include "tasks.hh"
#include "CanopyHydrology.hh"
......@@ -46,20 +46,24 @@ void top_level_task(const Task *task,
//
// grid cell x pft data for phenology
auto phenology_fs_names = std::vector<std::string>{ "elai", "esai" };
Data2D phenology(n_grid_cells, n_pfts, n_parts, "phenology", phenology_fs_names,
ctx, runtime);
Data<2> phenology("phenology", ctx, runtime,
Point<2>(n_grid_cells, n_pfts), Point<2>(n_parts,1),
phenology_fs_names);
// n_times_max x n_grid_cells forcing data
auto forcing_fs_ids = std::vector<std::string>{
auto forcing_fs_names = std::vector<std::string>{
"forc_rain", "forc_snow", "forc_air_temp", "forc_irrig"};
Data2D_Transposed forcing(n_grid_cells, n_times_max, n_parts, "forcing",
forcing_fs_ids, ctx, runtime);
Data<2> forcing("forcing", ctx, runtime,
Point<2>(n_times_max, n_grid_cells), Point<2>(1,n_parts),
forcing_fs_names);
// grid cell x pft water state and flux outputs
auto flux_fs_ids = std::vector<std::string>{
auto flux_fs_names = std::vector<std::string>{
"qflx_prec_intr", "qflx_irrig", "qflx_prec_grnd", "qflx_snwcp_liq",
"qflx_snwcp_ice", "qflx_snow_grnd_patch", "qflx_rain_grnd", "h2ocan"};
Data2D flux(n_grid_cells, n_pfts, n_parts, "flux", flux_fs_ids, ctx, runtime);
Data<2> flux("flux", ctx, runtime,
Point<2>(n_grid_cells, n_pfts), Point<2>(n_parts,1),
flux_fs_names);
// -----------------------------------------------------------------------------
// Initialization Phase
......@@ -124,12 +128,6 @@ int main(int argc, char **argv)
InitPhenology::preregister();
SumMinMaxReduction::preregister();
Runtime::preregister_projection_functor(Data2D::projection_id,
new Data2D::LocalProjectionFunction());
Runtime::preregister_projection_functor(Data2D_Transposed::projection_id,
new Data2D_Transposed::LocalProjectionFunction());
return Runtime::start(argc, argv);
}
......
......@@ -57,8 +57,8 @@ CanopyHydrology_kern1_multiple: test_CanopyHydrology_kern1_multiple
./test_CanopyHydrology_kern1_multiple &> test_CanopyHydrology_kern1_multiple.stdout
sandbox: test_sandbox
./sandbox
sandbox: test_sandbox_domain_template_magic
./test_sandbox_domain_template_magic
# If we're using CUDA we have to link with nvcc
test_%: %.cc.o $(GEN_OBJS) $(GEN_GPU_OBJS) $(SLIB_LEGION) $(SLIB_REALM)
......
#ifndef DOMAINS_HH_
#define DOMAINS_HH_
#include "legion.h"
using namespace Legion;
// projection
template<size_t DIM>
class LocalProjectionFunction : public ProjectionFunctor {
public:
LocalProjectionFunction(const Point<DIM>& partition, Runtime* rt) :
ProjectionFunctor(rt),
partition_(partition) {}
virtual ~LocalProjectionFunction() {}
public:
virtual LogicalRegion project(const Mappable *mappable, unsigned index,
LogicalRegion upper_bound,
const DomainPoint &point) override {
assert(false);
}
virtual LogicalRegion project(const Mappable *mappable, unsigned index,
LogicalPartition upper_bound,
const DomainPoint &point) override {
auto color = Point<DIM,coord_t>::ZEROES();
size_t j = 0;
for (size_t i = 0; i!=DIM; ++i) {
if (partition_[i] > 1) {
assert(j <= point.get_dim());
color[i] = point[j];
j++;
}
}
return runtime->get_logical_subregion_by_color(upper_bound, (DomainPoint) color);
}
virtual bool is_exclusive(void) const override { return true; }
virtual unsigned get_depth(void) const override { return 0; }
private:
Point<DIM> partition_;
};
template<size_t DIM>
struct Data {
Data(const std::string& name_,
Context ctx_, Runtime* runtime_) :
name(name_),
ctx(ctx_),
runtime(runtime_)
{
field_space = runtime->create_field_space(ctx);
allocator_ = runtime->create_field_allocator(ctx, field_space);
}
Data(const std::string& name_,
Context ctx_, Runtime* runtime_,
const Point<DIM> shape_,
const Point<DIM> blocked_shape_) :
Data(name_, ctx_, runtime_)
{
initShape(shape_, blocked_shape_);
}
Data(const std::string& name_,
Context ctx_, Runtime* runtime_,
const Point<DIM> shape_,
const Point<DIM> blocked_shape_,
const std::vector<std::string>& field_names_) :
Data(name_, ctx_, runtime_, shape_, blocked_shape_)
{
addFields<double>(field_names_);
finalize();
}
// member functions
void initShape(const Point<DIM> shape_, const Point<DIM> blocked_shape_)
{
shape = shape_;
blocked_shape = blocked_shape_;
domain = Rect<DIM,coord_t>(Point<DIM>::ZEROES(), shape - Point<DIM>::ONES());
index_space = runtime->create_index_space<DIM>(ctx, domain);
color_domain = Rect<DIM>(Point<DIM>::ZEROES(), blocked_shape - Point<DIM>::ONES());
Point<DIM> block;
for (int i = 0; i!=DIM; ++i) block[i] = shape[i] / blocked_shape[i];
index_partition = runtime->create_partition_by_blockify<DIM>(ctx, index_space, block);
runtime->attach_name(index_partition, (std::string("index_partition_")+name).c_str());
}
template<typename T>
FieldID allocateField_() {
return allocator_.allocate_field(sizeof(T), AUTO_GENERATE_ID);
}
template<typename T>
void addField(const std::string& fieldname) {
FieldID fid = allocateField_<T>();
field_ids[fieldname] = fid;
field_names.push_back(fieldname);
}
template<typename T>
void addFields(const std::vector<std::string>& fieldnames) {
for (auto fname : fieldnames) {
addField<T>(fname);
}
}
template<typename... Ts>
void addVariadicFields(const std::vector<std::string>& fieldnames) {
// unpack the template parameter, adding the right types
FieldID fids[]{allocateField_<Ts>()...};
size_t i = 0;
for (auto fname : fieldnames) {
field_ids[fname] = fids[i];
field_names.push_back(fname);
i++;
}
}
void finalize() {
allocator_ = FieldAllocator();
runtime->attach_name(field_space, (std::string("field_space_")+name).c_str());
// // DEBUG CRUFT
// std::vector<FieldID> my_fields;
// runtime->get_field_space_fields(ctx, field_space, my_fields);
// std::cout << "Allocated field space of size: " << my_fields.size() << std::endl;
// std::cout << "num ids: " << field_ids.size() << std::endl;
// std::cout << "num names: " << field_names.size() << std::endl;
// // END DEBUG CRUFT
// create logical regions, partitions
logical_region = runtime->create_logical_region(ctx, index_space, field_space);
runtime->attach_name(logical_region, (std::string("logical_region_")+name).c_str());
logical_partition = runtime->get_logical_partition(ctx, logical_region, index_partition);
runtime->attach_name(logical_partition, (std::string("logical_partition_")+name).c_str());
// register the projection
projection_id = runtime->generate_dynamic_projection_id();
auto* pf = new LocalProjectionFunction<DIM>(blocked_shape, runtime);
runtime->register_projection_functor(projection_id,pf);
}
~Data() {
runtime->destroy_logical_partition(ctx, logical_partition);
runtime->destroy_logical_region(ctx, logical_region);
runtime->destroy_field_space(ctx, field_space);
runtime->destroy_index_partition(ctx, index_partition);
runtime->destroy_index_space(ctx, index_space);
}
// name the container
std::string name;
Context ctx;
Runtime* runtime;
// index space
Point<DIM> shape;
Point<DIM> blocked_shape;
Rect<DIM> domain;
IndexSpaceT<DIM> index_space;
// index partition
coord_t n_parts;
Rect<DIM> color_domain;
IndexPartitionT<DIM, coord_t> index_partition;
// field space
std::vector<std::string> field_names;
std::map<std::string, FieldID> field_ids;
FieldSpace field_space;
FieldAllocator allocator_;
// cross product of index and field
LogicalRegion logical_region;
LogicalPartition logical_partition;
ProjectionID projection_id;
};
#endif
#ifndef DOMAINS_HH_
#define DOMAINS_HH_
#include "legion.h"
using namespace Legion;
namespace ProjectionIDs {
enum ProjectionIDs {
IDENTITY,
TO_1D_FROM_2D,
TO_1D_FROM_2D_TRANSPOSED
};
} // namespace
struct Data1D {
Data1D(coord_t n_grid_cells_, coord_t n_parts_,
const std::vector<std::string>& field_names_,
const std::string& name_,
Context ctx_, Runtime* runtime_) :
name(name_),
field_names(field_names_),
n_grid_cells(n_grid_cells_),
domain(Point<1>(0), Point<1>(n_grid_cells_-1)),
n_parts(n_parts_),
color_domain(Point<1>(0), Point<1>(n_parts_-1)),
ctx(ctx_),
runtime(runtime_)
{
index_space = runtime->create_index_space<1, coord_t>(ctx, domain);
auto block = Point<1>(n_grid_cells / n_parts);
index_partition = runtime->create_partition_by_blockify<1,coord_t>(ctx, index_space, block);
runtime->attach_name(index_partition, (std::string("index_partition_")+name).c_str());
field_space = runtime->create_field_space(ctx);
{
FieldAllocator allocator = runtime->create_field_allocator(ctx, field_space);
for (auto fname : field_names)
field_ids[fname] = allocator.allocate_field(sizeof(double), AUTO_GENERATE_ID);
}
runtime->attach_name(field_space, (std::string("field_space_")+name).c_str());
logical_region = runtime->create_logical_region(ctx, index_space, field_space);
runtime->attach_name(logical_region, (std::string("logical_region_")+name).c_str());
logical_partition = runtime->get_logical_partition(ctx, logical_region, index_partition);
runtime->attach_name(logical_partition, (std::string("logical_partition_")+name).c_str());
}
~Data1D() {
runtime->destroy_logical_partition(ctx, logical_partition);
runtime->destroy_logical_region(ctx, logical_region);
runtime->destroy_field_space(ctx, field_space);
runtime->destroy_index_partition(ctx, index_partition);
runtime->destroy_index_space(ctx, index_space);
}
// name the container
std::string name;
std::vector<std::string> field_names;
// index space and partition
coord_t n_grid_cells;
const Rect<1> domain;
IndexSpaceT<1,coord_t> index_space;
coord_t n_parts;
const Rect<1> color_domain;
IndexPartitionT<1, coord_t> index_partition;
// field space
std::map<std::string, unsigned> field_ids;
FieldSpace field_space;
// cross product of index and field
LogicalRegion logical_region;
LogicalPartition logical_partition;
// projection from this color_space into the 1D default color_space
const static unsigned projection_id = ProjectionIDs::IDENTITY;
Context ctx;
Runtime* runtime;
};
struct Data2D {
Data2D(coord_t n_grid_cells_, coord_t n_second_, coord_t n_parts_,
const std::string& name_,
const std::vector<std::string>& field_names_,
Context ctx_, Runtime* runtime_) :
name(name_),
field_names(field_names_),
n_grid_cells(n_grid_cells_),
n_second(n_second_),
domain(Point<2>(0,0), Point<2>(n_grid_cells_-1, n_second_-1)),
n_parts(n_parts_),
color_domain(Point<2>(0,0), Point<2>(n_parts_-1,0)),
ctx(ctx_),
runtime(runtime_)
{
index_space = runtime->create_index_space<2,coord_t>(ctx, domain);
auto block = Point<2>(n_grid_cells / n_parts, n_second);
index_partition = runtime->create_partition_by_blockify<2,coord_t>(ctx, index_space, block);
runtime->attach_name(index_partition, (std::string("index_partition_")+name).c_str());
field_space = runtime->create_field_space(ctx);
{
FieldAllocator allocator = runtime->create_field_allocator(ctx, field_space);
for (auto fname : field_names_)
field_ids[fname] = allocator.allocate_field(sizeof(double), AUTO_GENERATE_ID);
}
runtime->attach_name(field_space, (std::string("field_space_")+name).c_str());
logical_region = runtime->create_logical_region(ctx, index_space, field_space);
runtime->attach_name(logical_region, (std::string("logical_region_")+name).c_str());
logical_partition = runtime->get_logical_partition(ctx, logical_region, index_partition);
runtime->attach_name(logical_partition, (std::string("logical_partition_")+name).c_str());
}
~Data2D() {
runtime->destroy_logical_partition(ctx, logical_partition);
runtime->destroy_logical_region(ctx, logical_region);
runtime->destroy_field_space(ctx, field_space);
runtime->destroy_index_partition(ctx, index_partition);
runtime->destroy_index_space(ctx, index_space);
}
std::string name;
std::vector<std::string> field_names;
coord_t n_grid_cells, n_second;
const Rect<2> domain;
IndexSpaceT<2,coord_t> index_space;
IndexPartitionT<2, coord_t> index_partition;
std::map<std::string, unsigned> field_ids;
FieldSpace field_space;
LogicalRegion logical_region;
LogicalPartition logical_partition;
coord_t n_parts;
const Rect<2> color_domain;
Context ctx;
Runtime* runtime;
class LocalProjectionFunction : public ProjectionFunctor{
public:
LocalProjectionFunction() {}
LocalProjectionFunction(Runtime *rt) {}
virtual ~LocalProjectionFunction() {}
public:
virtual LogicalRegion project(const Mappable *mappable, unsigned index,
LogicalRegion upper_bound,
const DomainPoint &point) override {
assert(false);
}
virtual LogicalRegion project(const Mappable *mappable, unsigned index,
LogicalPartition upper_bound,
const DomainPoint &point) override {
Point<2,coord_t> color; color[0] = point[0]; color[1] = 0;
return runtime->get_logical_subregion_by_color(upper_bound, color);
}
virtual bool is_exclusive(void) const override { return true; }
virtual unsigned get_depth(void) const override { return 0; }
};
const static unsigned projection_id = ProjectionIDs::TO_1D_FROM_2D;
};
struct Data2D_Transposed {
Data2D_Transposed(coord_t n_grid_cells_, coord_t n_second_, coord_t n_parts_,
const std::string& name_,
const std::vector<std::string>& field_names_,
Context ctx_, Runtime* runtime_) :
name(name_),
field_names(field_names_),
n_grid_cells(n_grid_cells_),
n_second(n_second_),
domain(Point<2>(0,0), Point<2>(n_second_-1, n_grid_cells_-1)),
n_parts(n_parts_),
color_domain(Point<2>(0,0), Point<2>(0,n_parts_-1)),
ctx(ctx_),
runtime(runtime_)
{
index_space = runtime->create_index_space<2,coord_t>(ctx, domain);
auto block = Point<2>(n_second, n_grid_cells / n_parts);
index_partition = runtime->create_partition_by_blockify<2,coord_t>(ctx, index_space, block);
runtime->attach_name(index_partition, (std::string("index_partition_")+name).c_str());
field_space = runtime->create_field_space(ctx);
{
FieldAllocator allocator = runtime->create_field_allocator(ctx, field_space);
for (auto fname : field_names_)
field_ids[fname] = allocator.allocate_field(sizeof(double), AUTO_GENERATE_ID);
std::cout << "Field IDs (" << name_ << "):";
for (auto fid : field_ids) {
std::cout << fid.first << "," << fid.second << ";";
}
std::cout << std::endl;
}
runtime->attach_name(field_space, (std::string("field_space_")+name).c_str());
logical_region = runtime->create_logical_region(ctx, index_space, field_space);
runtime->attach_name(logical_region, (std::string("logical_region_")+name).c_str());
logical_partition = runtime->get_logical_partition(ctx, logical_region, index_partition);
runtime->attach_name(logical_partition, (std::string("logical_partition_")+name).c_str());
}
~Data2D_Transposed() {
runtime->destroy_logical_partition(ctx, logical_partition);
runtime->destroy_logical_region(ctx, logical_region);
runtime->destroy_field_space(ctx, field_space);
runtime->destroy_index_partition(ctx, index_partition);
runtime->destroy_index_space(ctx, index_space);
}
std::string name;
std::vector<std::string> field_names;
coord_t n_grid_cells, n_second;
const Rect<2> domain;
IndexSpaceT<2,coord_t> index_space;
IndexPartitionT<2, coord_t> index_partition;
std::map<std::string,unsigned> field_ids;
FieldSpace field_space;
LogicalRegion logical_region;
LogicalPartition logical_partition;
coord_t n_parts;
const Rect<2> color_domain;
static const unsigned projection_id = ProjectionIDs::TO_1D_FROM_2D_TRANSPOSED;
Context ctx;
Runtime* runtime;
class LocalProjectionFunction : public ProjectionFunctor{
public:
LocalProjectionFunction() {}
LocalProjectionFunction(Runtime *rt) {}
virtual ~LocalProjectionFunction() {}
public:
virtual LogicalRegion project(const Mappable *mappable, unsigned index,
LogicalRegion upper_bound,
const DomainPoint &point) override {
assert(false);
}
virtual LogicalRegion project(const Mappable *mappable, unsigned index,
LogicalPartition upper_bound,
const DomainPoint &point) override {
Point<2,coord_t> color; color[0] = 0; color[1] = point[0];
return runtime->get_logical_subregion_by_color(upper_bound, color);
}
virtual bool is_exclusive(void) const override { return true; }
virtual unsigned get_depth(void) const override { return 0; }
};
};
#endif
......@@ -12,7 +12,7 @@ using namespace Legion;
Future
SumMinMaxReduction::launch(Context ctx, Runtime *runtime,
Data2D& domain, const std::string& fname)
Data<2>& domain, const std::string& fname)
{
TaskLauncher accumlate_launcher(taskid, TaskArgument());
accumlate_launcher.add_region_requirement(
......@@ -70,7 +70,7 @@ std::string SumMinMaxReduction::name = "sum_min_max_reduction";
//
// =============================================================================
Future
InitPhenology::launch(Context ctx, Runtime *runtime, Data2D& data)
InitPhenology::launch(Context ctx, Runtime *runtime, Data<2>& data)
{
TaskLauncher phenology_launcher(taskid, TaskArgument(NULL, 0));
phenology_launcher.add_region_requirement(
......@@ -130,7 +130,7 @@ std::string InitPhenology::name = "init_phenology";
//
// =============================================================================
Future
InitForcing::launch(Context ctx, Runtime *runtime, Data2D_Transposed& data)
InitForcing::launch(Context ctx, Runtime *runtime, Data<2>& data)
{
std::cout << "LOG: Launching Init Forcing" << std::endl;
......@@ -209,9 +209,9 @@ std::string InitForcing::name = "init_forcing";
FutureMap
CanopyHydrology_Interception::launch(Context ctx, Runtime *runtime,
Rect<1>& color_space,
Data2D& phenology,
Data2D_Transposed& forcing,
Data2D& flux,
Data<2>& phenology,
Data<2>& forcing,
Data<2>& flux,
int itime)
{
// launch task to call interception
......
......@@ -39,7 +39,7 @@
#include <array>
#include "legion.h"
#include "domains.hh"
#include "data.hh"
using namespace Legion;
......@@ -62,7 +62,7 @@ enum TaskIDs {
// global on the full region!
struct SumMinMaxReduction {
Future launch(Context ctx, Runtime *runtime,
Data2D& domain, const std::string& fname);
Data<2>& domain, const std::string& fname);
static std::array<double,3> cpu_execute_task(const Task *task,
const std::vector<PhysicalRegion> &regions,
Context ctx, Runtime *runtime);
......@@ -77,7 +77,7 @@ struct SumMinMaxReduction {
//
// If this grows, will need to make it index-launched.
struct InitPhenology {
Future launch(Context ctx, Runtime *runtime, Data2D& data);
Future launch(Context ctx, Runtime *runtime, Data<2>& data);
static void cpu_execute_task(const Task *task,
const std::vector<PhysicalRegion> &regions,
Context ctx, Runtime *runtime);
......@@ -92,7 +92,7 @@ struct InitPhenology {
//
// If this grows, will need to make it index-launched.
struct InitForcing {
Future launch(Context ctx, Runtime *runtime, Data2D_Transposed& data);
Future launch(Context ctx, Runtime *runtime, Data<2>& data);
static int cpu_execute_task(const Task *task,
const std::vector<PhysicalRegion> &regions,