Tpetra parallel linear algebra Version of the Day
Loading...
Searching...
No Matches
Tpetra_Core.cpp
1// @HEADER
2// ***********************************************************************
3//
4// Tpetra: Templated Linear Algebra Services Package
5// Copyright (2008) Sandia Corporation
6//
7// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
8// the U.S. Government retains certain rights in this software.
9//
10// Redistribution and use in source and binary forms, with or without
11// modification, are permitted provided that the following conditions are
12// met:
13//
14// 1. Redistributions of source code must retain the above copyright
15// notice, this list of conditions and the following disclaimer.
16//
17// 2. Redistributions in binary form must reproduce the above copyright
18// notice, this list of conditions and the following disclaimer in the
19// documentation and/or other materials provided with the distribution.
20//
21// 3. Neither the name of the Corporation nor the names of the
22// contributors may be used to endorse or promote products derived from
23// this software without specific prior written permission.
24//
25// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36//
37// ************************************************************************
38// @HEADER
39
40#include "Tpetra_Core.hpp"
41#include "Tpetra_Details_mpiIsInitialized.hpp"
42
43#ifdef HAVE_TPETRACORE_MPI
44# include <Teuchos_DefaultMpiComm.hpp> // this includes mpi.h too
45#endif // HAVE_TPETRACORE_MPI
46#include <Teuchos_DefaultSerialComm.hpp>
47
48#include <Kokkos_Core.hpp>
49#include "Tpetra_Details_checkLaunchBlocking.hpp"
50
51namespace Tpetra {
52
53 namespace { // (anonymous)
54
55 class HideOutputExceptOnProcess0 {
56 public:
57 HideOutputExceptOnProcess0 (std::ostream& stream,
58 const int myRank) :
59 stream_ (stream),
60 originalBuffer_ (stream.rdbuf ())
61 {
62 if (myRank != 0) {
63 stream.rdbuf (blackHole_.rdbuf ());
64 }
65 }
66
67 ~HideOutputExceptOnProcess0 () {
68 stream_.rdbuf (originalBuffer_);
69 }
70 private:
71 std::ostream& stream_;
72 decltype (std::cout.rdbuf ()) originalBuffer_;
73 Teuchos::oblackholestream blackHole_;
74 };
75
76#if defined(HAVE_TPETRACORE_MPI)
77 bool mpiIsInitializedAndNotFinalized ()
78 {
79 int isInitialized = 0;
80 int isFinalized = 0;
81 // Not sure if MPI_Initialized or MPI_Finalized meet the strong
82 // exception guarantee.
83 try {
84 (void) MPI_Initialized (&isInitialized);
85 }
86 catch (...) {
87 isInitialized = 0;
88 }
89 try {
90 (void) MPI_Finalized (&isFinalized);
91 }
92 catch (...) {
93 isFinalized = 0;
94 }
95 return isInitialized != 0 && isFinalized == 0;
96 }
97
98 int getRankHarmlessly (MPI_Comm comm)
99 {
100 int myRank = 0;
101 if (mpiIsInitializedAndNotFinalized ()) {
102 try {
103 (void) MPI_Comm_rank (comm, &myRank);
104 }
105 catch (...) {
106 // Not sure if MPI_Comm_rank meets strong exception guarantee
107 myRank = 0;
108 }
109 }
110 return myRank;
111 }
112#endif // defined(HAVE_TPETRACORE_MPI)
113
114
115 // Whether one of the Tpetra::initialize() functions has been called before.
116 bool tpetraIsInitialized_ = false;
117
118 // Whether Tpetra initialized Kokkos. Tpetra::finalize only
119 // finalizes Kokkos if it initialized Kokkos. Otherwise,
120 // something else initialized Kokkos and is responsible for
121 // finalizing it.
122 bool tpetraInitializedKokkos_ = false;
123
124#ifdef HAVE_TPETRACORE_MPI
125 // Whether Tpetra initialized MPI. Tpetra::finalize only
126 // finalizes MPI if it initialized MPI. Otherwise, something else
127 // initialized MPI and is responsible for finalizing it.
128 bool tpetraInitializedMpi_ = false;
129#endif // HAVE_TPETRACORE_MPI
130
131 // Tpetra's default communicator, wrapped in a Teuchos wrapper.
132 // After Tpetra::finalize() is called, this GOES AWAY (is set to null).
133 Teuchos::RCP<const Teuchos::Comm<int> > wrappedDefaultComm_;
134
135 // This takes the same arguments as (the first two of) initialize().
136 void initKokkosIfNeeded (int* argc, char*** argv, const int myRank)
137 {
138 if (! tpetraInitializedKokkos_) {
139 // Kokkos doesn't have a global is_initialized(). However,
140 // Kokkos::initialize() always initializes the default execution
141 // space, so it suffices to check whether that was initialized.
142 const bool kokkosIsInitialized =
143 Kokkos::is_initialized ();
144 if (! kokkosIsInitialized) {
145 HideOutputExceptOnProcess0 hideCerr (std::cerr, myRank);
146 HideOutputExceptOnProcess0 hideCout (std::cout, myRank);
147
148 // Unlike MPI_Init, Kokkos promises not to modify argc and argv.
149 Kokkos::initialize (*argc, *argv);
150 tpetraInitializedKokkos_ = true;
151 }
152 }
153 Details::checkOldCudaLaunchBlocking();
154 const bool kokkosIsInitialized =
155 Kokkos::is_initialized ();
156 TEUCHOS_TEST_FOR_EXCEPTION
157 (! kokkosIsInitialized, std::logic_error, "At the end of "
158 "initKokkosIfNeeded, Kokkos is not initialized. "
159 "Please report this bug to the Tpetra developers.");
160 }
161
162#ifdef HAVE_TPETRACORE_MPI
163 // This takes the same arguments as MPI_Init and the first two
164 // arguments of initialize().
165 void initMpiIfNeeded (int* argc, char*** argv)
166 {
167 // Both MPI_Initialized and MPI_Finalized report true after
168 // MPI_Finalize has been called. It's not legal to call
169 // MPI_Init after MPI_Finalize has been called (see MPI 3.0
170 // Standard, Section 8.7). It would be unusual for users to
171 // want to use Tpetra after MPI_Finalize has been called, but
172 // there's no reason why we should forbid it. It just means
173 // that Tpetra will need to run without MPI.
174
175 const bool mpiReady = mpiIsInitializedAndNotFinalized ();
176 if (! mpiReady) {
177 // Tpetra doesn't currently need to call MPI_Init_thread,
178 // since with Tpetra, only one thread ever calls MPI
179 // functions. If we ever want to explore
180 // MPI_THREAD_MULTIPLE, here would be the place to call
181 // MPI_Init_thread.
182 const int err = MPI_Init (argc, argv);
183 TEUCHOS_TEST_FOR_EXCEPTION
184 (err != MPI_SUCCESS, std::runtime_error, "MPI_Init failed with "
185 "error code " << err << " != MPI_SUCCESS. If MPI was set up "
186 "correctly, then this should not happen, since we have already "
187 "checked that MPI_Init (or MPI_Init_thread) has not yet been "
188 "called. This may indicate that your MPI library is corrupted "
189 "or that it is incorrectly linked to your program.");
190 tpetraInitializedMpi_ = true;
191 }
192 }
193#endif // HAVE_TPETRACORE_MPI
194
195 } // namespace (anonymous)
196
199 }
200
201 Teuchos::RCP<const Teuchos::Comm<int> > getDefaultComm ()
202 {
203 // It's technically not correct to call this function if Tpetra
204 // has not yet been initialized, but requiring that may break some
205 // downstream tests.
206 //
207 // This function initializes wrappedDefaultComm_ lazily.
208 // Tpetra::initialize should not set it up.
209 if (wrappedDefaultComm_.is_null ()) {
210 Teuchos::RCP<const Teuchos::Comm<int> > comm;
211#ifdef HAVE_TPETRACORE_MPI
212 // Teuchos::MpiComm's constructor used to invoke MPI collectives.
213 // It still reserves the right to do so. This means MPI must be
214 // initialized and not finalized.
216 if (mpiReady) {
217 comm = Teuchos::rcp (new Teuchos::MpiComm<int> (MPI_COMM_WORLD));
218 }
219 else {
220 comm = Teuchos::rcp (new Teuchos::SerialComm<int> ());
221 }
222#else
223 comm = Teuchos::rcp (new Teuchos::SerialComm<int> ());
224#endif // HAVE_TPETRACORE_MPI
225 wrappedDefaultComm_ = comm;
226 }
227 return wrappedDefaultComm_;
228 }
229
230 void initialize (int* argc, char*** argv)
231 {
232 if (! tpetraIsInitialized_) {
233#if defined(HAVE_TPETRACORE_MPI)
235 // It's technically legal to initialize Tpetra after
236 // MPI_Finalize has been called. This means that we can't call
237 // MPI_Comm_rank without first checking MPI_Finalized.
239#else
240 const int myRank = 0;
241#endif // defined(HAVE_TPETRACORE_MPI)
243 }
245 }
246
247#ifdef HAVE_TPETRACORE_MPI
248 void initialize (int* argc, char*** argv, MPI_Comm comm)
249 {
250 if (! tpetraIsInitialized_) {
251#if defined(HAVE_TPETRACORE_MPI)
252 initMpiIfNeeded (argc, argv);
253 // It's technically legal to initialize Tpetra after
254 // MPI_Finalize has been called. This means that we can't call
255 // MPI_Comm_rank without first checking MPI_Finalized.
256 const int myRank = getRankHarmlessly (comm);
257#else
258 const int myRank = 0;
259#endif // defined(HAVE_TPETRACORE_MPI)
260 initKokkosIfNeeded (argc, argv, myRank);
261 }
262 tpetraIsInitialized_ = true;
263
264 // Set the default communicator. We set it here, after the above
265 // initialize() call, just in case users have not yet initialized
266 // MPI. (This is legal if users pass in a predefined
267 // communicator, like MPI_COMM_WORLD or MPI_COMM_SELF.)
268 //
269 // What if users have already called initialize() before, but with
270 // a different default communicator? There are two possible
271 // things we could do here:
272 //
273 // 1. Test via MPI_Comm_compare whether comm differs from the
274 // raw MPI communicator in wrappedDefaultComm_ (if indeed it
275 // is an MpiComm).
276 // 2. Accept that the user might want to change the default
277 // communicator, and let them do it.
278 //
279 // I prefer #2. Perhaps it would be sensible to print a warning
280 // here, but on which process? Would we use the old or the new
281 // communicator to find that process' rank? We don't want to use
282 // MPI_COMM_WORLD's Process 0, since neither communicator might
283 // include that process. Furthermore, in some environments, only
284 // Process 0 in MPI_COMM_WORLD is allowed to do I/O. Thus, we
285 // just let the change go without a warning.
286 wrappedDefaultComm_ = Teuchos::rcp (new Teuchos::MpiComm<int> (comm));
287 }
288#endif // HAVE_TPETRACORE_MPI
289
290 void
291 initialize (int* argc, char*** argv,
292 const Teuchos::RCP<const Teuchos::Comm<int> >& comm)
293 {
294 if (! tpetraIsInitialized_) {
295#if defined(HAVE_TPETRACORE_MPI)
297#endif // defined(HAVE_TPETRACORE_MPI)
298 // It's technically legal to initialize Tpetra after
299 // MPI_Finalize has been called. This means that we can't call
300 // MPI_Comm_rank without first checking MPI_Finalized.
301 const int myRank = comm->getRank ();
303 }
305 wrappedDefaultComm_ = comm;
306 }
307
308 void finalize ()
309 {
310 if (! tpetraIsInitialized_) {
311 return; // user didn't call initialize(), so do nothing at all
312 }
313
314 // Tpetra should only finalize Kokkos if it initialized Kokkos.
315 // See Github Issue #434.
316 if (tpetraInitializedKokkos_ && !Kokkos::is_finalized()) {
317 Kokkos::finalize ();
318 }
319
320 // Make sure that no outstanding references to the communicator
321 // remain. If users gave initialize() an MPI_Comm, _they_ are
322 // responsible for freeing it before calling finalize().
323 wrappedDefaultComm_ = Teuchos::null;
324
325#ifdef HAVE_TPETRACORE_MPI
326 // Tpetra should only finalize MPI if it initialized MPI.
327 // See Github Issue #434.
329 // finalize() is a kind of destructor, so it's a bad idea to
330 // throw an exception on error. MPI implementations do have
331 // the option to throw on error, so let's catch that here.
332 try {
334 // This must be called by the same thread that called
335 // MPI_Init or MPI_Init_thread (possibly, but not
336 // necessarily, in Tpetra::initialize()).
337 (void) MPI_Finalize ();
338 }
339 }
340 catch (...) {}
341 }
342#endif // HAVE_TPETRACORE_MPI
343
344 tpetraIsInitialized_ = false; // it's not anymore.
345 }
346
348 {
350 }
351
352#ifdef HAVE_TPETRA_MPI
353 ScopeGuard::ScopeGuard (int* argc, char*** argv, MPI_Comm comm)
354 {
356 }
357#endif // HAVE_TPETRA_MPI
358
363
364} // namespace Tpetra
Functions for initializing and finalizing Tpetra.
Struct that holds views of the contents of a CrsMatrix.
ScopeGuard()=delete
Default constructor (FORBIDDEN)
~ScopeGuard()
Finalize Tpetra.
bool mpiIsInitialized()
Has MPI_Init been called (on this process)?
Namespace Tpetra contains the class and methods constituting the Tpetra library.
void initialize(int *argc, char ***argv)
Initialize Tpetra.
bool isInitialized()
Whether Tpetra is in an initialized state.
void finalize()
Finalize Tpetra.
Teuchos::RCP< const Teuchos::Comm< int > > getDefaultComm()
Get Tpetra's default communicator.