Rheolef  7.1
an efficient C++ finite element environment
geo_mpi_partition.cc
Go to the documentation of this file.
1 // call to parmetis
22 
23 #include "rheolef/config.h"
24 #ifdef _RHEOLEF_HAVE_MPI
25 #include "rheolef/geo_element.h"
26 #include "rheolef/hack_array.h"
27 
28 #if defined(_RHEOLEF_HAVE_SCOTCH)
29 #include "geo_partition_scotch.h"
30 #elif defined(_RHEOLEF_HAVE_PARMETIS)
31 #include <parmetis.h>
32 typedef idx_t idxtype; // change from previous parmetis versions
33 #endif // _RHEOLEF_HAVE_PARMETIS
34 
35 namespace rheolef {
36 
37 disarray<size_t>
40  ios_geo_element,
41  const distributor& ios_ownership, // by_dimension
42  size_t map_dim,
43  size_t dis_nv)
44 {
46  typedef hack_array<geo_element_hack>::const_iterator const_iterator_by_variant;
47 
48  communicator comm = ios_ownership.comm();
49  size_type nproc = comm.size();
50  size_type my_proc = comm.rank();
51 
52  disarray<size_type> partition (ios_ownership);
53  size_type iproc = 0;
54  if (partition.dis_size() <= nproc || dis_nv <= nproc) {
55  // mesh is very small (nelts <= nproc): parmetis does not support it !
56  for (size_type ie = 0, nie = ios_ownership.size(); ie < nie; ie++) {
57  partition [ie] = my_proc;
58  }
59  return partition;
60  }
61  // here, mesh is not so small: sufficient for parmetis
62  disarray<idxtype> part (ios_ownership);
63  std::vector<idxtype> elmdist (nproc+1);
64  std::copy (
65  ios_ownership.begin(),
66  ios_ownership.end(),
67  elmdist.begin());
68  std::vector<idxtype> eptr (ios_ownership.dis_size()+1);
69  {
70  size_type ie = 0;
71  eptr [0] = 0;
74  for (const_iterator_by_variant iter = ios_geo_element [variant].begin(), last = ios_geo_element [variant].end();
75  iter != last; iter++, ie++) {
76  const geo_element& K = *iter;
77  eptr [ie+1] = eptr[ie] + K.size();
78  }
79  }
80  }
81  std::vector<idxtype> eind (eptr [ios_ownership.size()]);
82  std::vector<idxtype>::iterator iter_eind = eind.begin();
83  {
86  for (const_iterator_by_variant iter = ios_geo_element [variant].begin(), last = ios_geo_element [variant].end();
87  iter != last; iter++) {
88  const geo_element& K = *iter;
89  for (size_type iloc = 0; iloc < K.size(); iloc++, iter_eind++) {
90  *iter_eind = K[iloc];
91  }
92  }
93  }
94  }
95  idxtype wgtflag = 0;
96  idxtype numflag = 0;
97  idxtype mgcnum = map_dim;
98  idxtype ncon = 1;
99  idxtype nparts = nproc;
100  std::vector<float> tpwgts (nparts*ncon);
101  std::fill (tpwgts.begin(), tpwgts.end(), 1./nparts);
102  float ubvec [12];
103  std::fill (ubvec, ubvec+ncon, 1.05);
104  idxtype options [10];
105  options[0] = 1;
106  const int pvm3_option_dbglvl = 1;
107  const int pvm3_option_seed = 2;
108  options[pvm3_option_dbglvl] = 0; // otherwise: timming print to stdout...
109  options[pvm3_option_seed] = 0;
110  idxtype edgecut;
111  MPI_Comm raw_comm = comm;
112  idxtype *elmwgt = 0;
113 #if defined(_RHEOLEF_HAVE_SCOTCH)
115  elmdist.begin().operator->(),
116  eptr.begin().operator->(),
117  eind,
118  elmwgt,
119  &ncon,
120  &mgcnum,
121  &nparts,
122  tpwgts.begin().operator->(),
123  ubvec,
124  &edgecut,
125  part.begin().operator->(),
126  comm);
127 #elif defined(_RHEOLEF_HAVE_PARMETIS)
128  ParMETIS_V3_PartMeshKway(
129  elmdist.begin().operator->(),
130  eptr.begin().operator->(),
131  eind.begin().operator->(),
132  NULL,
133  &wgtflag,
134  &numflag,
135  &ncon,
136  &mgcnum,
137  &nparts,
138  tpwgts.begin().operator->(),
139  ubvec,
140  options,
141  &edgecut,
142  part.begin().operator->(),
143  &raw_comm);
144 #else // _RHEOLEF_HAVE_PARMETIS
145 # error either parmetis nor scotch partitioner founded
146 #endif // _RHEOLEF_HAVE_PARMETIS
147 
148  // convert int to size_type (64 bits : not the same sizeof...)
149  std::copy (part.begin(), part.end(), partition.begin());
150  return partition;
151 }
152 
153 } // namespace rheolef
154 #endif // _RHEOLEF_HAVE_MPI
rheolef::reference_element::last_variant_by_dimension
static variant_type last_variant_by_dimension(size_type dim)
Definition: reference_element.h:150
rheolef::idxtype
my_idxtype idxtype
Definition: geo_partition_scotch.h:30
rheolef::distributor::comm
const communicator_type & comm() const
Definition: distributor.h:145
rheolef::Vector::end
iterator end()
Definition: Vector.h:225
rheolef::hack_array::const_iterator
std::vector< T, A >::const_iterator const_iterator
Definition: hack_array.h:351
rheolef::distributor
see the distributor page for the full documentation
Definition: distributor.h:62
rheolef::geo_mpi_partition
disarray< size_t > geo_mpi_partition(const std::array< hack_array< geo_element_hack >, reference_element::max_variant > &ios_geo_element, const distributor &ownership_by_dimension, size_t map_dim, size_t dis_nv)
Definition: geo_mpi_partition.cc:38
rheolef::geo_element::size
size_type size() const
Definition: geo_element.h:168
rheolef::geo_element
see the geo_element page for the full documentation
Definition: geo_element.h:102
rheolef::size_type
size_t size_type
Definition: basis_get.cc:76
rheolef::reference_element::first_variant_by_dimension
static variant_type first_variant_by_dimension(size_type dim)
Definition: reference_element.h:148
mkgeo_ball.variant
int variant
Definition: mkgeo_ball.sh:149
rheolef::disarray::size_type
rep::base::size_type size_type
Definition: disarray.h:463
rheolef::hack_array
Definition: hack_array.h:346
rheolef
This file is part of Rheolef.
Definition: compiler_eigen.h:37
rheolef::distributor::dis_size
size_type dis_size() const
global and local sizes
Definition: distributor.h:207
rheolef::reference_element::max_variant
static const variant_type max_variant
Definition: reference_element.h:82
rheolef::disarray< size_type >
rheolef::Vector::begin
iterator begin()
Definition: Vector.h:217
rheolef::geo_partition_scotch
void geo_partition_scotch(my_idxtype *elmdist, my_idxtype *eptr, vector< my_idxtype > &eind, my_idxtype *elmwgt, int *ncon, int *ncommonnodes, int *nparts, float *tpwgts, float *ubvec, int *edgecut, my_idxtype *part, const mpi::communicator &comm)
Definition: geo_mpi_partition_scotch.cc:48
size_type
field::size_type size_type
Definition: branch.cc:425
mkgeo_ball.map_dim
map_dim
Definition: mkgeo_ball.sh:337
geo_partition_scotch.h
rheolef::distributor::size
size_type size(size_type iproc) const
Definition: distributor.h:163