Go to the documentation of this file.
3 #ifndef DUNE_GRID_YASPGRID_TORUS_HH
4 #define DUNE_GRID_YASPGRID_TORUS_HH
17 #include <dune/common/binaryfunctions.hh>
18 #include <dune/common/streamoperators.hh>
43 template<
class CollectiveCommunication,
int d>
77 for (
int i=0; i<d; i++)
84 if (inc != _comm.size())
85 DUNE_THROW(Dune::Exception,
"Communicator size and result of the given load balancer do not match!");
122 CollectiveCommunication
comm ()
const
136 for (
int i=d-1; i>=0; i--)
137 if (c[i]<0 || c[i]>=_dims[i])
return false;
146 for (
int i=d-1; i>=0; i--)
157 for (
int i=0; i<d; i++)
coord[i] =
coord[i]%_dims[i];
159 for (
int i=0; i<d; i++)
rank +=
coord[i]*_increment[i];
167 coord[dir] = (
coord[dir]+_dims[dir]+cnt)%_dims[dir];
178 for (
int i=0; i<d; i++)
180 if (
coord[i]%2==1) c += power;
185 for (
int i=0; i<d; i++)
187 if (_dims[i]>1 &&
coord[i]==_dims[i]-1) c += power;
204 for (
int i=0; i<d; ++i)
214 for (
int i=0; i<d; i++)
219 if (
coord[i]==0 && periodic[i]==
false)
return false;
224 if (
coord[i]==_dims[i]-1 && periodic[i]==
false)
return false;
244 for (
int i=0; i<d; i++)
247 int m = size_in[i]/_dims[i];
248 int r = size_in[i]%_dims[i];
252 if (
coord[i]<_dims[i]-r)
254 origin_out[i] = origin_in[i] +
coord[i]*m;
260 origin_out[i] = origin_in[i] + (_dims[i]-r)*m + (
coord[i]-(_dims[i]-r))*(m+1);
265 return maxsize/(sz/_comm.size());
305 for (
int j=0; j<d; ++j)
306 dist += std::abs(
delta[j]);
331 typename std::deque<CommPartner>::const_iterator i;
337 return ProcListIterator(_sendlist.begin());
343 return ProcListIterator(_sendlist.end());
349 return ProcListIterator(_recvlist.begin());
355 return ProcListIterator(_recvlist.end());
363 task.buffer = buffer;
365 if (
rank!=_comm.rank())
366 _sendrequests.push_back(task);
368 _localsendrequests.push_back(task);
376 task.buffer = buffer;
378 if (
rank!=_comm.rank())
379 _recvrequests.push_back(task);
381 _localrecvrequests.push_back(task);
388 if (_localsendrequests.size()!=_localrecvrequests.size())
390 std::cout <<
"[" <<
rank() <<
"]: ERROR: local sends/receives do not match in exchange!" << std::endl;
393 for (
unsigned int i=0; i<_localsendrequests.size(); i++)
395 if (_localsendrequests[i].size!=_localrecvrequests[i].size)
397 std::cout <<
"[" <<
rank() <<
"]: ERROR: size in local sends/receive does not match in exchange!" << std::endl;
400 memcpy(_localrecvrequests[i].buffer,_localsendrequests[i].buffer,_localsendrequests[i].size);
402 _localsendrequests.clear();
403 _localrecvrequests.clear();
408 std::vector<MPI_Request> requests(_sendrequests.size() + _recvrequests.size());
409 MPI_Request* req = requests.data();
412 for (
unsigned int i=0; i<_sendrequests.size(); i++)
417 MPI_Isend(_sendrequests[i].buffer, _sendrequests[i].size, MPI_BYTE,
418 _sendrequests[i].
rank, _tag, _comm, req++);
422 for (
unsigned int i=0; i<_recvrequests.size(); i++)
427 MPI_Irecv(_recvrequests[i].buffer, _recvrequests[i].size, MPI_BYTE,
428 _recvrequests[i].
rank, _tag, _comm, req++);
432 MPI_Waitall(requests.size(), requests.data(), MPI_STATUSES_IGNORE);
435 _sendrequests.clear();
436 _recvrequests.clear();
444 _comm.template allreduce<Dune::Max<double>,
double>(&x, &res, 1);
451 s <<
"[" <<
rank() <<
"]: Torus " <<
procs() <<
" processor(s) arranged as " <<
dims() << std::endl;
454 s <<
"[" <<
rank() <<
"]: send to "
455 <<
"rank=" << i.rank()
456 <<
" index=" << i.index()
457 <<
" delta=" << i.delta() <<
" dist=" << i.distance() << std::endl;
461 s <<
"[" <<
rank() <<
"]: recv from "
462 <<
"rank=" << i.rank()
463 <<
" index=" << i.index()
464 <<
" delta=" << i.delta() <<
" dist=" << i.distance() << std::endl;
476 std::fill(delta.begin(), delta.end(), -1);
485 for (
int i=0; i<d; i++)
486 nb[i] = ( me[i]+_dims[i]+delta[i] ) % _dims[i];
492 for (
int i=0; i<d; i++)
498 _recvlist.push_back(cp);
499 cp.index = last-index;
500 _sendlist.push_front(cp);
507 for (
int i=0; i<d; i++)
522 CollectiveCommunication _comm;
527 std::deque<CommPartner> _sendlist;
528 std::deque<CommPartner> _recvlist;
530 mutable std::vector<CommTask> _sendrequests;
531 mutable std::vector<CommTask> _recvrequests;
532 mutable std::vector<CommTask> _localsendrequests;
533 mutable std::vector<CommTask> _localrecvrequests;
538 template <
class CollectiveCommunication,
int d>
This file provides tools to partition YaspGrids. If you want to write your own partitioner,...
const iTupel & dims() const
return dimensions of torus
Definition: torus.hh:110
ProcListIterator sendend() const
end of send list
Definition: torus.hh:341
double global_max(double x) const
global max
Definition: torus.hh:441
virtual void loadbalance(const iTupel &, int, iTupel &) const =0
bool is_neighbor(iTupel delta, std::bitset< d > periodic) const
return true if neighbor with given delta is a neighbor under the given periodicity
Definition: torus.hh:210
int color(int rank) const
assign color to given rank
Definition: torus.hh:195
CollectiveCommunication comm() const
return communicator
Definition: torus.hh:122
void send(int rank, void *buffer, int size) const
store a send request; buffers are sent in order; handles also local requests with memcpy
Definition: torus.hh:359
iTupel coord() const
return own coordinates
Definition: torus.hh:98
bool operator!=(const ProcListIterator &iter)
Return true when two iterators do not point to same member.
Definition: torus.hh:318
std::ostream & operator<<(std::ostream &out, const PartitionType &type)
write a PartitionType to a stream
Definition: gridenums.hh:70
iTupel delta() const
return distance vector
Definition: torus.hh:289
ProcListIterator(typename std::deque< CommPartner >::const_iterator iter)
make an iterator
Definition: torus.hh:277
int rank_relative(int rank, int dir, int cnt) const
return rank of process where its coordinate in direction dir has offset cnt (handles periodic case)
Definition: torus.hh:164
Torus()
constructor making uninitialized object
Definition: torus.hh:65
void exchange() const
exchange messages stored in request buffers; clear request buffers afterwards
Definition: torus.hh:385
double partition(int rank, iTupel origin_in, iTupel size_in, iTupel &origin_out, iTupel &size_out) const
partition the given grid onto the torus and return the piece of the process with given rank; returns ...
Definition: torus.hh:237
ProcListIterator recvend() const
last process in receive list
Definition: torus.hh:353
ProcListIterator sendbegin() const
first process in send list
Definition: torus.hh:335
ProcListIterator recvbegin() const
first process in receive list
Definition: torus.hh:347
Torus(CollectiveCommunication comm, int tag, iTupel size, const YLoadBalance< d > *lb)
make partitioner from communicator and coarse mesh size
Definition: torus.hh:69
int tag() const
return tag used by torus
Definition: torus.hh:128
int dims(int i) const
return dimensions of torus in direction i
Definition: torus.hh:116
ProcListIterator & operator++()
Increment iterator to next cell.
Definition: torus.hh:324
int distance() const
return 1-norm of distance vector
Definition: torus.hh:301
bool operator==(const ProcListIterator &iter)
Return true when two iterators point to same member.
Definition: torus.hh:311
int rank() const
return own rank
Definition: torus.hh:92
int coord_to_rank(iTupel coord) const
map coordinate in torus to rank using lexicographic ordering
Definition: torus.hh:155
bool inside(iTupel c) const
return true if coordinate is inside torus
Definition: torus.hh:134
std::array< int, d > iTupel
type used to pass tupels in and out
Definition: torus.hh:47
void print(std::ostream &s) const
print contents of torus object
Definition: torus.hh:449
int color(const iTupel &coord) const
assign color to given coordinate
Definition: torus.hh:172
iTupel rank_to_coord(int rank) const
map rank to coordinate in torus using lexicographic ordering
Definition: torus.hh:142
a base class for the yaspgrid partitioning strategy The name might be irritating. It will probably ch...
Definition: partitioning.hh:23
int rank() const
return rank of neighboring process
Definition: torus.hh:283
int neighbors() const
return the number of neighbors, which is
Definition: torus.hh:201
void recv(int rank, void *buffer, int size) const
store a receive request; buffers are received in order; handles also local requests with memcpy
Definition: torus.hh:372
int procs() const
return number of processes
Definition: torus.hh:104
Include standard header files.
Definition: agrid.hh:58
int index() const
return index in proclist
Definition: torus.hh:295