Kokkos Core Kernels Package
Version of the Day
core
src
Kokkos_Atomic.hpp
Go to the documentation of this file.
1
/*
2
//@HEADER
3
// ************************************************************************
4
//
5
// Kokkos v. 3.0
6
// Copyright (2020) National Technology & Engineering
7
// Solutions of Sandia, LLC (NTESS).
8
//
9
// Under the terms of Contract DE-NA0003525 with NTESS,
10
// the U.S. Government retains certain rights in this software.
11
//
12
// Redistribution and use in source and binary forms, with or without
13
// modification, are permitted provided that the following conditions are
14
// met:
15
//
16
// 1. Redistributions of source code must retain the above copyright
17
// notice, this list of conditions and the following disclaimer.
18
//
19
// 2. Redistributions in binary form must reproduce the above copyright
20
// notice, this list of conditions and the following disclaimer in the
21
// documentation and/or other materials provided with the distribution.
22
//
23
// 3. Neither the name of the Corporation nor the names of the
24
// contributors may be used to endorse or promote products derived from
25
// this software without specific prior written permission.
26
//
27
// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
28
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
30
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
31
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
32
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
33
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
34
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
35
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
36
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
37
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38
//
39
// Questions? Contact Christian R. Trott (crtrott@sandia.gov)
40
//
41
// ************************************************************************
42
//@HEADER
43
*/
44
67
68
#ifndef KOKKOS_ATOMIC_HPP
69
#define KOKKOS_ATOMIC_HPP
70
71
#include <Kokkos_Macros.hpp>
72
#include <Kokkos_HostSpace.hpp>
73
#include <impl/Kokkos_Traits.hpp>
74
75
//----------------------------------------------------------------------------
76
#if defined(_WIN32)
77
#define KOKKOS_ENABLE_WINDOWS_ATOMICS
78
#else
79
#if defined(KOKKOS_ENABLE_CUDA)
80
81
// Compiling NVIDIA device code, must use Cuda atomics:
82
83
#define KOKKOS_ENABLE_CUDA_ATOMICS
84
85
#elif defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_ROCM_GPU)
86
87
#define KOKKOS_ENABLE_ROCM_ATOMICS
88
89
#elif defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HIP_GPU)
90
91
#define KOKKOS_ENABLE_HIP_ATOMICS
92
93
#endif
94
95
#if !defined(KOKKOS_ENABLE_GNU_ATOMICS) && \
96
!defined(KOKKOS_ENABLE_INTEL_ATOMICS) && \
97
!defined(KOKKOS_ENABLE_OPENMP_ATOMICS) && \
98
!defined(KOKKOS_ENABLE_STD_ATOMICS) && \
99
!defined(KOKKOS_ENABLE_SERIAL_ATOMICS)
100
101
// Compiling for non-Cuda atomic implementation has not been pre-selected.
102
// Choose the best implementation for the detected compiler.
103
// Preference: GCC, INTEL, OMP31
104
105
#if defined(KOKKOS_INTERNAL_NOT_PARALLEL)
106
107
#define KOKKOS_ENABLE_SERIAL_ATOMICS
108
109
#elif defined(KOKKOS_COMPILER_GNU) || defined(KOKKOS_COMPILER_CLANG) || \
110
(defined(KOKKOS_COMPILER_NVCC))
111
112
#define KOKKOS_ENABLE_GNU_ATOMICS
113
114
#elif defined(KOKKOS_COMPILER_INTEL) || defined(KOKKOS_COMPILER_CRAYC)
115
116
#define KOKKOS_ENABLE_INTEL_ATOMICS
117
118
#elif defined(_OPENMP) && (201107 <= _OPENMP)
119
120
#define KOKKOS_ENABLE_OPENMP_ATOMICS
121
122
#else
123
124
#error "KOKKOS_ATOMICS_USE : Unsupported compiler"
125
126
#endif
127
128
#endif
/* Not pre-selected atomic implementation */
129
#endif
130
131
#ifdef KOKKOS_ENABLE_CUDA
132
#include <Cuda/Kokkos_Cuda_Locks.hpp>
133
#endif
134
135
namespace
Kokkos
{
136
template
<
typename
T>
137
KOKKOS_INLINE_FUNCTION
void
atomic_add(
volatile
T*
const
dest,
const
T src);
138
139
// Atomic increment
140
template
<
typename
T>
141
KOKKOS_INLINE_FUNCTION
void
atomic_increment(
volatile
T* a);
142
143
template
<
typename
T>
144
KOKKOS_INLINE_FUNCTION
void
atomic_decrement(
volatile
T* a);
145
}
// namespace Kokkos
146
147
namespace
Kokkos
{
148
149
inline
const
char
* atomic_query_version() {
150
#if defined(KOKKOS_ENABLE_CUDA_ATOMICS)
151
return
"KOKKOS_ENABLE_CUDA_ATOMICS"
;
152
#elif defined(KOKKOS_ENABLE_GNU_ATOMICS)
153
return
"KOKKOS_ENABLE_GNU_ATOMICS"
;
154
#elif defined(KOKKOS_ENABLE_INTEL_ATOMICS)
155
return
"KOKKOS_ENABLE_INTEL_ATOMICS"
;
156
#elif defined(KOKKOS_ENABLE_OPENMP_ATOMICS)
157
return
"KOKKOS_ENABLE_OPENMP_ATOMICS"
;
158
#elif defined(KOKKOS_ENABLE_WINDOWS_ATOMICS)
159
return
"KOKKOS_ENABLE_WINDOWS_ATOMICS"
;
160
#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS)
161
return
"KOKKOS_ENABLE_SERIAL_ATOMICS"
;
162
#else
163
#error "No valid response for atomic_query_version!"
164
#endif
165
}
166
167
}
// namespace Kokkos
168
169
//----------------------------------------------------------------------------
170
// Atomic Memory Orders
171
//
172
// Implements Strongly-typed analogs of C++ standard memory orders
173
#include "impl/Kokkos_Atomic_Memory_Order.hpp"
174
175
#if defined(KOKKOS_ENABLE_ROCM)
176
namespace
Kokkos
{
177
namespace
Impl {
178
extern
KOKKOS_INLINE_FUNCTION
bool
lock_address_rocm_space(
void
* ptr);
179
180
extern
KOKKOS_INLINE_FUNCTION
void
unlock_address_rocm_space(
void
* ptr);
181
}
// namespace Impl
182
}
// namespace Kokkos
183
#include <ROCm/Kokkos_ROCm_Atomic.hpp>
184
#endif
185
#if defined(KOKKOS_ENABLE_HIP)
186
#include <HIP/Kokkos_HIP_Atomic.hpp>
187
#endif
188
189
#ifdef _WIN32
190
#include "impl/Kokkos_Atomic_Windows.hpp"
191
#else
192
//----------------------------------------------------------------------------
193
// Atomic Assembly
194
//
195
// Implements CAS128-bit in assembly
196
197
#include "impl/Kokkos_Atomic_Assembly.hpp"
198
199
//----------------------------------------------------------------------------
200
// Atomic exchange
201
//
202
// template< typename T >
203
// T atomic_exchange( volatile T* const dest , const T val )
204
// { T tmp = *dest ; *dest = val ; return tmp ; }
205
206
#include "impl/Kokkos_Atomic_Exchange.hpp"
207
208
//----------------------------------------------------------------------------
209
// Atomic compare-and-exchange
210
//
211
// template<class T>
212
// bool atomic_compare_exchange_strong(volatile T* const dest, const T compare,
213
// const T val) { bool equal = compare == *dest ; if ( equal ) { *dest = val ; }
214
// return equal ; }
215
216
#include "impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp"
217
218
#endif //_WIN32
219
220
#include "impl/Kokkos_Atomic_Generic.hpp"
221
222
#ifndef _WIN32
223
//----------------------------------------------------------------------------
224
// Atomic fetch and add
225
//
226
// template<class T>
227
// T atomic_fetch_add(volatile T* const dest, const T val)
228
// { T tmp = *dest ; *dest += val ; return tmp ; }
229
230
#include "impl/Kokkos_Atomic_Fetch_Add.hpp"
231
232
//----------------------------------------------------------------------------
233
// Atomic increment
234
//
235
// template<class T>
236
// T atomic_increment(volatile T* const dest)
237
// { dest++; }
238
239
#include "impl/Kokkos_Atomic_Increment.hpp"
240
241
//----------------------------------------------------------------------------
242
// Atomic Decrement
243
//
244
// template<class T>
245
// T atomic_decrement(volatile T* const dest)
246
// { dest--; }
247
248
#include "impl/Kokkos_Atomic_Decrement.hpp"
249
250
//----------------------------------------------------------------------------
251
// Atomic fetch and sub
252
//
253
// template<class T>
254
// T atomic_fetch_sub(volatile T* const dest, const T val)
255
// { T tmp = *dest ; *dest -= val ; return tmp ; }
256
257
#include "impl/Kokkos_Atomic_Fetch_Sub.hpp"
258
259
//----------------------------------------------------------------------------
260
// Atomic fetch and or
261
//
262
// template<class T>
263
// T atomic_fetch_or(volatile T* const dest, const T val)
264
// { T tmp = *dest ; *dest = tmp | val ; return tmp ; }
265
266
#include "impl/Kokkos_Atomic_Fetch_Or.hpp"
267
268
//----------------------------------------------------------------------------
269
// Atomic fetch and and
270
//
271
// template<class T>
272
// T atomic_fetch_and(volatile T* const dest, const T val)
273
// { T tmp = *dest ; *dest = tmp & val ; return tmp ; }
274
275
#include "impl/Kokkos_Atomic_Fetch_And.hpp"
276
277
//----------------------------------------------------------------------------
278
// Atomic MinMax
279
//
280
// template<class T>
281
// T atomic_min(volatile T* const dest, const T val)
282
// { T tmp = *dest ; *dest = min(*dest, val); return tmp ; }
283
// template<class T>
284
// T atomic_max(volatile T* const dest, const T val)
285
// { T tmp = *dest ; *dest = max(*dest, val); return tmp ; }
286
287
#include "impl/Kokkos_Atomic_MinMax.hpp"
288
#endif
/*Not _WIN32*/
289
290
//----------------------------------------------------------------------------
291
// Memory fence
292
//
293
// All loads and stores from this thread will be globally consistent before
294
// continuing
295
//
296
// void memory_fence() {...};
297
#include "impl/Kokkos_Memory_Fence.hpp"
298
299
//----------------------------------------------------------------------------
300
// Provide volatile_load and safe_load
301
//
302
// T volatile_load(T const volatile * const ptr);
303
//
304
// T const& safe_load(T const * const ptr);
305
// XEON PHI
306
// T safe_load(T const * const ptr
307
308
#include "impl/Kokkos_Volatile_Load.hpp"
309
310
//----------------------------------------------------------------------------
311
// Provide atomic loads and stores with memory order semantics
312
313
#include "impl/Kokkos_Atomic_Load.hpp"
314
#include "impl/Kokkos_Atomic_Store.hpp"
315
316
// Generic functions using the above defined functions
317
#include "impl/Kokkos_Atomic_Generic_Secondary.hpp"
318
//----------------------------------------------------------------------------
319
// This atomic-style macro should be an inlined function, not a macro
320
321
#if defined(KOKKOS_COMPILER_GNU) && !defined(__PGIC__) && \
322
!defined(__CUDA_ARCH__)
323
324
#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) __builtin_prefetch(addr, 0, 0)
325
#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) __builtin_prefetch(addr, 1, 0)
326
327
#else
328
329
#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) ((void)0)
330
#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) ((void)0)
331
332
#endif
333
334
//----------------------------------------------------------------------------
335
336
#endif
/* KOKKOS_ATOMIC_HPP */
Kokkos
Definition:
dummy.cpp:3
Generated by
1.8.14