Tpetra parallel linear algebra Version of the Day
Loading...
Searching...
No Matches
Tpetra_Distributor.cpp
1// ***********************************************************************
2//
3// Tpetra: Templated Linear Algebra Services Package
4// Copyright (2008) Sandia Corporation
5//
6// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7// the U.S. Government retains certain rights in this software.
8//
9// Redistribution and use in source and binary forms, with or without
10// modification, are permitted provided that the following conditions are
11// met:
12//
13// 1. Redistributions of source code must retain the above copyright
14// notice, this list of conditions and the following disclaimer.
15//
16// 2. Redistributions in binary form must reproduce the above copyright
17// notice, this list of conditions and the following disclaimer in the
18// documentation and/or other materials provided with the distribution.
19//
20// 3. Neither the name of the Corporation nor the names of the
21// contributors may be used to endorse or promote products derived from
22// this software without specific prior written permission.
23//
24// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
25// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
28// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
29// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
30// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
31// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
32// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
33// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
34// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35//
36// ************************************************************************
37// @HEADER
38
39#include "Tpetra_Distributor.hpp"
42#include "Tpetra_Util.hpp"
43#include "Tpetra_Details_makeValidVerboseStream.hpp"
44#include "Teuchos_StandardParameterEntryValidators.hpp"
45#include "Teuchos_VerboseObjectParameterListHelpers.hpp"
46#include <numeric>
47
48namespace Tpetra {
49 // We set default values of Distributor's Boolean parameters here,
50 // in this one place. That way, if we want to change the default
51 // value of a parameter, we don't have to search the whole file to
52 // ensure a consistent setting.
53 namespace {
54 // Default value of the "Debug" parameter.
55 const bool tpetraDistributorDebugDefault = false;
56 } // namespace (anonymous)
57
58 Teuchos::Array<std::string>
60 {
61 Teuchos::Array<std::string> sendTypes;
62 sendTypes.push_back ("Isend");
63 sendTypes.push_back ("Send");
64 sendTypes.push_back ("Alltoall");
65 return sendTypes;
66 }
67
69 Distributor (const Teuchos::RCP<const Teuchos::Comm<int> >& comm,
70 const Teuchos::RCP<Teuchos::FancyOStream>& /* out */,
71 const Teuchos::RCP<Teuchos::ParameterList>& plist)
72 : plan_(comm)
73 {
74 this->setParameterList(plist);
75 }
76
78 Distributor (const Teuchos::RCP<const Teuchos::Comm<int> >& comm)
79 : Distributor (comm, Teuchos::null, Teuchos::null)
80 {}
81
83 Distributor (const Teuchos::RCP<const Teuchos::Comm<int> >& comm,
84 const Teuchos::RCP<Teuchos::FancyOStream>& out)
85 : Distributor (comm, out, Teuchos::null)
86 {}
87
89 Distributor (const Teuchos::RCP<const Teuchos::Comm<int> >& comm,
90 const Teuchos::RCP<Teuchos::ParameterList>& plist)
91 : Distributor (comm, Teuchos::null, plist)
92 {}
93
96 : plan_(distributor.plan_)
97 , actor_(distributor.actor_)
98 , verbose_ (distributor.verbose_)
99 , reverseDistributor_ (distributor.reverseDistributor_)
100 {
101 using Teuchos::ParameterList;
102 using Teuchos::RCP;
103 using Teuchos::rcp;
104
105 RCP<const ParameterList> rhsList = distributor.getParameterList ();
106 RCP<ParameterList> newList = rhsList.is_null () ? Teuchos::null :
107 Teuchos::parameterList (*rhsList);
108 this->setParameterList (newList);
109 }
110
112 using Teuchos::ParameterList;
113 using Teuchos::parameterList;
114 using Teuchos::RCP;
115
116 std::swap (plan_, rhs.plan_);
117 std::swap (actor_, rhs.actor_);
118 std::swap (verbose_, rhs.verbose_);
119 std::swap (reverseDistributor_, rhs.reverseDistributor_);
120
121 // Swap parameter lists. If they are the same object, make a deep
122 // copy first, so that modifying one won't modify the other one.
124 RCP<ParameterList> rhsList = rhs.getNonconstParameterList ();
125 if (lhsList.getRawPtr () == rhsList.getRawPtr () && ! rhsList.is_null ()) {
127 }
128 if (! rhsList.is_null ()) {
129 this->setMyParamList (rhsList);
130 }
131 if (! lhsList.is_null ()) {
132 rhs.setMyParamList (lhsList);
133 }
134
135 // We don't need to swap timers, because all instances of
136 // Distributor use the same timers.
137 }
138
139 bool
140 Distributor::getVerbose()
141 {
142 return Details::Behavior::verbose("Distributor") ||
143 Details::Behavior::verbose("Tpetra::Distributor");
144 }
145
146 std::unique_ptr<std::string>
147 Distributor::
148 createPrefix(const char methodName[]) const
149 {
151 plan_.getComm().getRawPtr(), "Distributor", methodName);
152 }
153
154 void
156 setParameterList (const Teuchos::RCP<Teuchos::ParameterList>& plist)
157 {
158 using ::Tpetra::Details::Behavior;
159 using Teuchos::FancyOStream;
160 using Teuchos::getIntegralValue;
161 using Teuchos::includesVerbLevel;
162 using Teuchos::ParameterList;
163 using Teuchos::parameterList;
164 using Teuchos::RCP;
165 using std::endl;
166
167 if (! plist.is_null()) {
169 plist->validateParametersAndSetDefaults (*validParams);
170
171 // ParameterListAcceptor semantics require pointer identity of the
172 // sublist passed to setParameterList(), so we save the pointer.
173 this->setMyParamList (plist);
174
176 planParams->remove("Debug", false);
177 planParams->remove("VerboseObject", false);
178 plan_.setParameterList(planParams);
179 }
180 }
181
182 Teuchos::RCP<const Teuchos::ParameterList>
184 {
185 using Teuchos::Array;
186 using Teuchos::ParameterList;
187 using Teuchos::parameterList;
188 using Teuchos::RCP;
189 using Teuchos::setStringToIntegralParameter;
190
191 const bool debug = tpetraDistributorDebugDefault;
192
194 const std::string defaultSendType ("Send");
196 sendTypeEnums.push_back (Details::DISTRIBUTOR_ISEND);
197 sendTypeEnums.push_back (Details::DISTRIBUTOR_SEND);
198 sendTypeEnums.push_back (Details::DISTRIBUTOR_ALLTOALL);
199
200 RCP<ParameterList> plist = parameterList ("Tpetra::Distributor");
202 defaultSendType, "When using MPI, the variant of send to use in "
203 "do[Reverse]Posts()", sendTypes(), sendTypeEnums(), plist.getRawPtr());
204 plist->set ("Debug", debug, "Whether to print copious debugging output on "
205 "all processes.");
206 plist->set ("Timer Label","","Label for Time Monitor output");
207
208 // mfh 24 Dec 2015: Tpetra no longer inherits from
209 // Teuchos::VerboseObject, so it doesn't need the "VerboseObject"
210 // sublist. However, we retain the "VerboseObject" sublist
211 // anyway, for backwards compatibility (otherwise the above
212 // validation would fail with an invalid parameter name, should
213 // the user still want to provide this list).
214 Teuchos::setupVerboseObjectSublist (&*plist);
215 return Teuchos::rcp_const_cast<const ParameterList> (plist);
216 }
217
218
220 { return plan_.getTotalReceiveLength(); }
221
223 { return plan_.getNumReceives(); }
224
226 { return plan_.hasSelfMessage(); }
227
229 { return plan_.getNumSends(); }
230
232 { return plan_.getMaxSendLength(); }
233
234 Teuchos::ArrayView<const int> Distributor::getProcsFrom() const
235 { return plan_.getProcsFrom(); }
236
237 Teuchos::ArrayView<const size_t> Distributor::getLengthsFrom() const
238 { return plan_.getLengthsFrom(); }
239
240 Teuchos::ArrayView<const int> Distributor::getProcsTo() const
241 { return plan_.getProcsTo(); }
242
243 Teuchos::ArrayView<const size_t> Distributor::getLengthsTo() const
244 { return plan_.getLengthsTo(); }
245
246 Teuchos::RCP<Distributor>
248 if (reverseDistributor_.is_null () && create) {
249 createReverseDistributor ();
250 }
252 (reverseDistributor_.is_null () && create, std::logic_error, "The reverse "
253 "Distributor is null after createReverseDistributor returned. "
254 "Please report this bug to the Tpetra developers.");
255 return reverseDistributor_;
256 }
257
258
259 void
260 Distributor::createReverseDistributor() const
261 {
262 reverseDistributor_ = Teuchos::rcp(new Distributor(plan_.getComm()));
263 reverseDistributor_->plan_ = *plan_.getReversePlan();
264 reverseDistributor_->verbose_ = verbose_;
265
266 // requests_: Allocated on demand.
267 // reverseDistributor_: See note below
268
269 // I am my reverse Distributor's reverse Distributor.
270 // Thus, it would be legit to do the following:
271 //
272 // reverseDistributor_->reverseDistributor_ = Teuchos::rcp (this, false);
273 //
274 // (Note use of a "weak reference" to avoid a circular RCP
275 // dependency.) The only issue is that if users hold on to the
276 // reverse Distributor but let go of the forward one, this
277 // reference won't be valid anymore. However, the reverse
278 // Distributor is really an implementation detail of Distributor
279 // and not meant to be used directly, so we don't need to do this.
280 reverseDistributor_->reverseDistributor_ = Teuchos::null;
281 }
282
283 void
285 {
286 actor_.doWaits(plan_);
287 }
288
290 // call doWaits() on the reverse Distributor, if it exists
291 if (! reverseDistributor_.is_null()) {
292 reverseDistributor_->doWaits();
293 }
294 }
295
296 std::string Distributor::description () const {
297 std::ostringstream out;
298
299 out << "\"Tpetra::Distributor\": {";
300 const std::string label = this->getObjectLabel ();
301 if (label != "") {
302 out << "Label: " << label << ", ";
303 }
304 out << "How initialized: "
305 << Details::DistributorHowInitializedEnumToString (plan_.howInitialized())
306 << ", Parameters: {"
307 << "Send type: "
308 << DistributorSendTypeEnumToString (plan_.getSendType())
309 << ", Debug: " << (verbose_ ? "true" : "false")
310 << "}}";
311 return out.str ();
312 }
313
314 std::string
315 Distributor::
316 localDescribeToString (const Teuchos::EVerbosityLevel vl) const
317 {
318 using Teuchos::toString;
319 using Teuchos::VERB_HIGH;
320 using Teuchos::VERB_EXTREME;
321 using std::endl;
322
323 // This preserves current behavior of Distributor.
324 if (vl <= Teuchos::VERB_LOW || plan_.getComm().is_null ()) {
325 return std::string ();
326 }
327
328 auto outStringP = Teuchos::rcp (new std::ostringstream ());
329 auto outp = Teuchos::getFancyOStream (outStringP); // returns RCP
330 Teuchos::FancyOStream& out = *outp;
331
332 const int myRank = plan_.getComm()->getRank ();
333 const int numProcs = plan_.getComm()->getSize ();
334 out << "Process " << myRank << " of " << numProcs << ":" << endl;
335 Teuchos::OSTab tab1 (out);
336
337 out << "selfMessage: " << hasSelfMessage() << endl;
338 out << "numSends: " << getNumSends() << endl;
339 if (vl == VERB_HIGH || vl == VERB_EXTREME) {
340 out << "procsTo: " << toString (plan_.getProcsTo()) << endl;
341 out << "lengthsTo: " << toString (plan_.getLengthsTo()) << endl;
342 out << "maxSendLength: " << getMaxSendLength() << endl;
343 }
344 if (vl == VERB_EXTREME) {
345 out << "startsTo: " << toString (plan_.getStartsTo()) << endl;
346 out << "indicesTo: " << toString (plan_.getIndicesTo()) << endl;
347 }
348 if (vl == VERB_HIGH || vl == VERB_EXTREME) {
349 out << "numReceives: " << getNumReceives() << endl;
350 out << "totalReceiveLength: " << getTotalReceiveLength() << endl;
351 out << "lengthsFrom: " << toString (plan_.getLengthsFrom()) << endl;
352 out << "procsFrom: " << toString (plan_.getProcsFrom()) << endl;
353 }
354
355 out.flush (); // make sure the ostringstream got everything
356 return outStringP->str ();
357 }
358
359 void
361 describe (Teuchos::FancyOStream& out,
362 const Teuchos::EVerbosityLevel verbLevel) const
363 {
364 using std::endl;
365 using Teuchos::VERB_DEFAULT;
366 using Teuchos::VERB_NONE;
367 using Teuchos::VERB_LOW;
368 using Teuchos::VERB_MEDIUM;
369 using Teuchos::VERB_HIGH;
370 using Teuchos::VERB_EXTREME;
371 const Teuchos::EVerbosityLevel vl =
373
374 if (vl == VERB_NONE) {
375 return; // don't print anything
376 }
377 // If this Distributor's Comm is null, then the the calling
378 // process does not participate in Distributor-related collective
379 // operations with the other processes. In that case, it is not
380 // even legal to call this method. The reasonable thing to do in
381 // that case is nothing.
382 if (plan_.getComm().is_null ()) {
383 return;
384 }
385 const int myRank = plan_.getComm()->getRank ();
386 const int numProcs = plan_.getComm()->getSize ();
387
388 // Only Process 0 should touch the output stream, but this method
389 // in general may need to do communication. Thus, we may need to
390 // preserve the current tab level across multiple "if (myRank ==
391 // 0) { ... }" inner scopes. This is why we sometimes create
392 // OSTab instances by pointer, instead of by value. We only need
393 // to create them by pointer if the tab level must persist through
394 // multiple inner scopes.
395 Teuchos::RCP<Teuchos::OSTab> tab0, tab1;
396
397 if (myRank == 0) {
398 // At every verbosity level but VERB_NONE, Process 0 prints.
399 // By convention, describe() always begins with a tab before
400 // printing.
401 tab0 = Teuchos::rcp (new Teuchos::OSTab (out));
402 // We quote the class name because it contains colons.
403 // This makes the output valid YAML.
404 out << "\"Tpetra::Distributor\":" << endl;
405 tab1 = Teuchos::rcp (new Teuchos::OSTab (out));
406
407 const std::string label = this->getObjectLabel ();
408 if (label != "") {
409 out << "Label: " << label << endl;
410 }
411 out << "Number of processes: " << numProcs << endl
412 << "How initialized: "
413 << Details::DistributorHowInitializedEnumToString (plan_.howInitialized())
414 << endl;
415 {
416 out << "Parameters: " << endl;
417 Teuchos::OSTab tab2 (out);
418 out << "\"Send type\": "
419 << DistributorSendTypeEnumToString (plan_.getSendType()) << endl
420 << "\"Debug\": " << (verbose_ ? "true" : "false") << endl;
421 }
422 } // if myRank == 0
423
424 // This is collective over the Map's communicator.
425 if (vl > VERB_LOW) {
426 const std::string lclStr = this->localDescribeToString (vl);
427 Tpetra::Details::gathervPrint (out, lclStr, *plan_.getComm());
428 }
429
430 out << "Reverse Distributor:";
431 if (reverseDistributor_.is_null ()) {
432 out << " null" << endl;
433 }
434 else {
435 out << endl;
436 reverseDistributor_->describe (out, vl);
437 }
438 }
439
440 size_t
442 createFromSends(const Teuchos::ArrayView<const int>& exportProcIDs)
443 {
444 return plan_.createFromSends(exportProcIDs);
445 }
446
447 void
449 createFromSendsAndRecvs (const Teuchos::ArrayView<const int>& exportProcIDs,
450 const Teuchos::ArrayView<const int>& remoteProcIDs)
451 {
452 plan_.createFromSendsAndRecvs(exportProcIDs, remoteProcIDs);
453 }
454
455} // namespace Tpetra
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra's behavior.
Declaration of a function that prints strings from each process.
Stand-alone utility functions and macros.
Struct that holds views of the contents of a CrsMatrix.
static bool verbose()
Whether Tpetra is in verbose mode.
Sets up and executes a communication plan for a Tpetra DistObject.
size_t getMaxSendLength() const
Maximum number of values this process will send to another single process.
Teuchos::RCP< Distributor > getReverse(bool create=true) const
A reverse communication plan Distributor.
Teuchos::ArrayView< const int > getProcsTo() const
Ranks of the processes to which this process will send values.
size_t getNumReceives() const
The number of processes from which we will receive data.
void setParameterList(const Teuchos::RCP< Teuchos::ParameterList > &plist)
Set Distributor parameters.
size_t getTotalReceiveLength() const
Total number of values this process will receive from other processes.
bool hasSelfMessage() const
Whether the calling process will send or receive messages to itself.
void swap(Distributor &rhs)
Swap the contents of rhs with those of *this.
Teuchos::ArrayView< const size_t > getLengthsTo() const
Number of values this process will send to each process.
Teuchos::ArrayView< const int > getProcsFrom() const
Ranks of the processes sending values to this process.
Distributor(const Teuchos::RCP< const Teuchos::Comm< int > > &comm)
Construct using the specified communicator and default parameters.
std::string description() const
Return a one-line description of this object.
size_t createFromSends(const Teuchos::ArrayView< const int > &exportProcIDs)
Set up Distributor using list of process ranks to which this process will send.
void createFromSendsAndRecvs(const Teuchos::ArrayView< const int > &exportProcIDs, const Teuchos::ArrayView< const int > &remoteProcIDs)
Set up Distributor using list of process ranks to which to send, and list of process ranks from which...
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters() const
List of valid Distributor parameters.
Teuchos::ArrayView< const size_t > getLengthsFrom() const
Number of values this process will receive from each process.
size_t getNumSends() const
The number of processes to which we will send data.
void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Describe this object in a human-readable way to the given output stream.
std::unique_ptr< std::string > createPrefix(const int myRank, const char prefix[])
Create string prefix for each line of verbose output.
std::string DistributorHowInitializedEnumToString(EDistributorHowInitialized how)
Convert an EDistributorHowInitialized enum value to a string.
void gathervPrint(std::ostream &out, const std::string &s, const Teuchos::Comm< int > &comm)
On Process 0 in the given communicator, print strings from each process in that communicator,...
Namespace Tpetra contains the class and methods constituting the Tpetra library.
Teuchos::Array< std::string > distributorSendTypes()
Valid values for Distributor's "Send type" parameter.