40#ifndef TPETRA_DETAILS_PACKCRSGRAPH_DEF_HPP
41#define TPETRA_DETAILS_PACKCRSGRAPH_DEF_HPP
43#include "TpetraCore_config.h"
44#include "Teuchos_Array.hpp"
45#include "Teuchos_ArrayView.hpp"
85namespace PackCrsGraphImpl {
93template<
class OutputOffsetsViewType,
95 class InputOffsetsViewType,
96 class InputLocalRowIndicesViewType,
97 class InputLocalRowPidsViewType,
99#ifdef HAVE_TPETRA_DEBUG
107 typedef typename OutputOffsetsViewType::non_const_value_type output_offset_type;
108 typedef typename CountsViewType::non_const_value_type count_type;
109 typedef typename InputOffsetsViewType::non_const_value_type input_offset_type;
110 typedef typename InputLocalRowIndicesViewType::non_const_value_type local_row_index_type;
111 typedef typename InputLocalRowPidsViewType::non_const_value_type local_row_pid_type;
113 typedef typename OutputOffsetsViewType::device_type device_type;
114 static_assert (std::is_same<
typename CountsViewType::device_type::execution_space,
115 typename device_type::execution_space>::value,
116 "OutputOffsetsViewType and CountsViewType must have the same execution space.");
117 static_assert (Kokkos::is_view<OutputOffsetsViewType>::value,
118 "OutputOffsetsViewType must be a Kokkos::View.");
119 static_assert (std::is_same<typename OutputOffsetsViewType::value_type, output_offset_type>::value,
120 "OutputOffsetsViewType must be a nonconst Kokkos::View.");
121 static_assert (std::is_integral<output_offset_type>::value,
122 "The type of each entry of OutputOffsetsViewType must be a built-in integer type.");
123 static_assert (Kokkos::is_view<CountsViewType>::value,
124 "CountsViewType must be a Kokkos::View.");
125 static_assert (std::is_same<typename CountsViewType::value_type, output_offset_type>::value,
126 "CountsViewType must be a nonconst Kokkos::View.");
127 static_assert (std::is_integral<count_type>::value,
128 "The type of each entry of CountsViewType must be a built-in integer type.");
129 static_assert (Kokkos::is_view<InputOffsetsViewType>::value,
130 "InputOffsetsViewType must be a Kokkos::View.");
131 static_assert (std::is_integral<input_offset_type>::value,
132 "The type of each entry of InputOffsetsViewType must be a built-in integer type.");
133 static_assert (Kokkos::is_view<InputLocalRowIndicesViewType>::value,
134 "InputLocalRowIndicesViewType must be a Kokkos::View.");
135 static_assert (std::is_integral<local_row_index_type>::value,
136 "The type of each entry of InputLocalRowIndicesViewType must be a built-in integer type.");
151 const size_t numRowsToPack =
static_cast<size_t> (lclRowInds_.extent (0));
153 if (
numRowsToPack !=
static_cast<size_t> (counts_.extent (0))) {
154 std::ostringstream
os;
156 <<
" != counts.extent(0) = " << counts_.extent (0)
161 static_cast<size_t> (outputOffsets_.extent (0))) {
162 std::ostringstream
os;
164 <<
" != outputOffsets.extent(0) = " << outputOffsets_.extent (0)
173 output_offset_type& update,
174 const bool final)
const
177 if (
curInd <
static_cast<local_row_index_type
> (0)) {
185 if (
curInd >=
static_cast<local_row_index_type
> (outputOffsets_.extent (0))) {
190 outputOffsets_(
curInd) = update;
193 if (
curInd <
static_cast<local_row_index_type
> (counts_.extent (0))) {
195 if (
static_cast<size_t> (
lclRow + 1) >=
static_cast<size_t> (rowOffsets_.extent (0)) ||
196 static_cast<local_row_index_type
> (
lclRow) <
static_cast<local_row_index_type
> (0)) {
204 const count_type count =
205 static_cast<count_type
> (rowOffsets_(
lclRow+1) - rowOffsets_(
lclRow));
210 ?
static_cast<count_type
>(0)
211 : count * (1 + (lclRowPids_.size() > 0 ? 1 : 0));
226 auto error_h = Kokkos::create_mirror_view (error_);
228 using execution_space =
typename device_type::execution_space;
229 Kokkos::deep_copy (execution_space(),
error_h, error_);
236 typename InputOffsetsViewType::const_type rowOffsets_;
237 typename InputLocalRowIndicesViewType::const_type lclRowInds_;
238 typename InputLocalRowPidsViewType::const_type lclRowPids_;
239 Kokkos::View<int, device_type> error_;
256typename CountsViewType::non_const_value_type
265 typename InputLocalRowIndicesViewType::const_type,
266 typename InputLocalRowPidsViewType::const_type>
functor_type;
267 typedef typename CountsViewType::non_const_value_type count_type;
268 typedef typename OutputOffsetsViewType::size_type size_type;
269 typedef typename OutputOffsetsViewType::execution_space execution_space;
270 typedef typename functor_type::local_row_index_type LO;
271 typedef Kokkos::RangePolicy<execution_space, LO> range_type;
272 const char prefix[] =
"computeNumPacketsAndOffsets: ";
274 count_type count = 0;
282 (
rowOffsets.extent (0) <=
static_cast<size_type
> (1),
283 std::invalid_argument,
prefix <<
"There is at least one row to pack, "
284 "but the graph has no rows. lclRowInds.extent(0) = " <<
289 static_cast<size_type
> (
numRowsToPack + 1), std::invalid_argument,
290 prefix <<
"Output dimension does not match number of rows to pack. "
292 <<
" != lclRowInds.extent(0) + 1 = "
305 (
errCode != 0, std::runtime_error,
prefix <<
"parallel_scan error code "
315 errStr = std::unique_ptr<std::ostringstream> (
new std::ostringstream ());
321 <<
total <<
"." << std::endl;
325 os <<
"outputOffsets: [";
332 os <<
"]" << std::endl;
340 os <<
"]" << std::endl;
354 using Tpetra::Details::getEntryOnHost;
355 return static_cast<count_type
> (getEntryOnHost (
outputOffsets,
378 const Kokkos::View<Packet*, BufferDeviceType>& exports,
383 const bool pack_pids)
385 using LO =
typename LocalMapType::local_ordinal_type;
386 using GO =
typename LocalMapType::global_ordinal_type;
390 return static_cast<size_t>(0);
421struct PackCrsGraphFunctor {
427 using num_packets_per_lid_view_type =
428 Kokkos::View<const size_t*, BufferDeviceType>;
429 using offsets_view_type = Kokkos::View<const size_t*, BufferDeviceType>;
430 using exports_view_type = Kokkos::View<Packet*, BufferDeviceType>;
431 using export_lids_view_type =
433 using source_pids_view_type =
437 typename num_packets_per_lid_view_type::non_const_value_type;
438 using offset_type =
typename offsets_view_type::non_const_value_type;
439 using value_type = Kokkos::pair<int, LO>;
441 static_assert (std::is_same<LO, typename local_graph_type::data_type>::value,
442 "local_map_type::local_ordinal_type and "
443 "local_graph_type::data_type must be the same.");
445 local_graph_type local_graph;
446 local_map_type local_col_map;
447 exports_view_type exports;
448 num_packets_per_lid_view_type num_packets_per_lid;
449 export_lids_view_type export_lids;
450 source_pids_view_type source_pids;
451 offsets_view_type offsets;
473 static_cast<LO
> (local_graph.row_map.extent (0));
476 std::logic_error,
"local_graph.row_map.extent(0) = "
482 using ::Tpetra::Details::OrdinalTraits;
486 KOKKOS_INLINE_FUNCTION
void
487 join (value_type& dst,
const value_type& src)
const
491 if (src.first != 0 && dst.first == 0) {
496 KOKKOS_INLINE_FUNCTION
497 void operator() (
const LO i, value_type& dst)
const
499 const size_t offset = offsets[i];
500 const LO export_lid = export_lids[i];
501 const size_t buf_size = exports.size();
502 const size_t num_packets_this_lid = num_packets_per_lid(i);
503 const size_t num_ent =
504 static_cast<size_t> (local_graph.row_map[export_lid+1]
505 - local_graph.row_map[export_lid]);
515 if (export_lid >=
static_cast<LO
>(local_graph.numRows())) {
516 if (dst.first != 0) {
517 dst = Kokkos::make_pair (1, i);
521 else if ((offset > buf_size || offset + num_packets_this_lid > buf_size)) {
522 if (dst.first != 0) {
523 dst = Kokkos::make_pair (2, i);
533 const auto row_beg = local_graph.row_map[export_lid];
534 const auto row_end = local_graph.row_map[export_lid + 1];
535 auto lids_in = Kokkos::subview (local_graph.entries,
536 Kokkos::make_pair (row_beg, row_end));
537 size_t num_ent_packed_this_row =
538 packRow (local_col_map, exports, lids_in,
539 source_pids, offset, num_ent, pack_pids);
540 if (num_ent_packed_this_row != num_packets_this_lid) {
541 if (dst.first != 0) {
542 dst = Kokkos::make_pair (3, i);
555template<
class Packet,
558 class BufferDeviceType>
562 const Kokkos::View<Packet*, BufferDeviceType>& exports,
565 >::input_array_type& num_packets_per_lid,
568 >::input_array_type& export_lids,
571 >::input_array_type& source_pids,
572 const Kokkos::View<const size_t*, BufferDeviceType>& offsets,
573 const bool pack_pids)
576 using execution_space =
typename LocalGraph::device_type::execution_space;
577 using range_type = Kokkos::RangePolicy<execution_space, LO>;
578 const char prefix[] =
"Tpetra::Details::PackCrsGraphImpl::do_pack: ";
580 if (export_lids.extent (0) != 0) {
582 (
static_cast<size_t> (offsets.extent (0)) !=
583 static_cast<size_t> (export_lids.extent (0) + 1),
584 std::invalid_argument,
prefix <<
"offsets.extent(0) = "
585 << offsets.extent (0) <<
" != export_lids.extent(0) (= "
586 << export_lids.extent (0) <<
") + 1.");
588 (export_lids.extent (0) != num_packets_per_lid.extent (0),
589 std::invalid_argument,
prefix <<
"export_lids.extent(0) = " <<
590 export_lids.extent (0) <<
" != num_packets_per_lid.extent(0) = "
591 << num_packets_per_lid.extent (0) <<
".");
596 (pack_pids && exports.extent (0) != 0 &&
597 source_pids.extent (0) == 0, std::invalid_argument,
prefix <<
598 "pack_pids is true, and exports.extent(0) = " <<
599 exports.extent (0) <<
" != 0, meaning that we need to pack at "
600 "least one graph entry, but source_pids.extent(0) = 0.");
607 num_packets_per_lid, export_lids,
608 source_pids, offsets, pack_pids);
610 typename pack_functor_type::value_type
result;
611 range_type
range (0, num_packets_per_lid.extent (0));
617 std::ostringstream
os;
619 os <<
"invalid local row index";
621 else if (
result.first == 2) {
622 os <<
"invalid offset";
625 (
true, std::runtime_error,
prefix <<
"PackCrsGraphFunctor "
626 "reported error code " <<
result.first <<
" (" <<
os.str ()
627 <<
") for the first bad row " <<
result.second <<
".");
657template<
typename LO,
typename GO,
typename NT>
668 >& num_packets_per_lid,
678 const bool pack_pids)
682 using packet_type =
typename crs_graph_type::packet_type;
683 using buffer_device_type =
typename crs_graph_type::buffer_device_type;
684 using exports_view_type = Kokkos::DualView<packet_type*, buffer_device_type>;
685 using local_graph_device_type =
typename crs_graph_type::local_graph_device_type;
687 const char prefix[] =
"Tpetra::Details::packCrsGraph: ";
688 constexpr bool debug =
false;
690 local_graph_device_type local_graph =
sourceGraph.getLocalGraphDevice ();
691 local_map_type local_col_map =
sourceGraph.getColMap ()->getLocalMap ();
701 std::invalid_argument,
prefix <<
"num_export_lids.extent(0) = "
703 << num_packets_per_lid.extent (0) <<
".");
706 (num_packets_per_lid.data () ==
nullptr, std::invalid_argument,
708 "num_packets_per_lid.data() = "
709 << num_packets_per_lid.data () <<
" == NULL.");
713 exports = exports_view_type (
"exports", 0);
727 if (count >
size_t (exports.extent (0))) {
728 exports = exports_view_type (
"exports", count);
730 std::ostringstream
os;
731 os <<
"*** exports resized to " << count << std::endl;
732 std::cerr <<
os.str ();
736 std::ostringstream
os;
737 os <<
"*** count: " << count <<
", exports.extent(0): "
738 << exports.extent (0) << std::endl;
739 std::cerr <<
os.str ();
746 (pack_pids && exports.extent (0) != 0 &&
748 "pack_pids is true, and exports.extent(0) = " <<
749 exports.extent (0) <<
" != 0, meaning that we need to pack at least "
750 "one graph entry, but export_pids.extent(0) = 0.");
752 exports.modify_device ();
755 (local_graph, local_col_map,
exports_d, num_packets_per_lid,
762template<
typename LO,
typename GO,
typename NT>
767 const Teuchos::ArrayView<const LO>&
exportLIDs,
770 using Kokkos::HostSpace;
771 using Kokkos::MemoryUnmanaged;
774 using packet_type =
typename crs_graph_type::packet_type;
775 using BDT =
typename crs_graph_type::buffer_device_type;
786 "num_packets_per_lid");
795 Kokkos::DualView<packet_type*, BDT>
exports_dv;
796 constexpr bool pack_pids =
false;
802 "num_packets_per_lid_d's non_const_value_type should be size_t.");
807 "num_packets_per_lid_d's BDT should be size_t.");
812 "export_lids_d's device_type should be BDT.");
817 "export_pids_d's non_const_value_type should be int.");
822 "export_pids_d's device_type should be BDT.");
824 PackCrsGraphImpl::packCrsGraph
835 using execution_space =
typename BDT::execution_space;
843 if (
static_cast<size_t> (exports.size ()) !=
844 static_cast<size_t> (
exports_dv.extent (0))) {
848 exports_h (exports.getRawPtr (), exports.size ());
855template<
typename LO,
typename GO,
typename NT>
858 const Kokkos::DualView<
862 const Kokkos::DualView<
872 > num_packets_per_lid,
874 const bool pack_pids)
878 using BDT =
typename crs_graph_type::buffer_device_type;
879 using PT =
typename crs_graph_type::packet_type;
881 using LGT =
typename crs_graph_type::local_graph_device_type;
882 using LMT =
typename crs_graph_type::map_type::local_map_type;
883 const char prefix[] =
"Tpetra::Details::packCrsGraphNew: ";
886 const LMT local_col_map =
sourceGraph.getColMap ()->getLocalMap ();
894 static_cast<size_t> (export_lids.extent (0));
897 static_cast<size_t> (num_packets_per_lid.extent (0)),
898 std::invalid_argument,
prefix <<
"num_export_lids.extent(0) = "
900 << num_packets_per_lid.extent (0) <<
".");
903 num_packets_per_lid.view_device ().data () ==
nullptr,
905 <<
" != 0, but num_packets_per_lid.view_device().data() = nullptr.");
913 using offsets_type = Kokkos::View<size_t*, BDT>;
918 num_packets_per_lid.clear_sync_state ();
919 num_packets_per_lid.modify_device ();
920 using PackCrsGraphImpl::computeNumPacketsAndOffsets;
922 computeNumPacketsAndOffsets (offsets, num_packets_per_lid.view_device (),
924 export_lids.view_device (),
928 if (count >
static_cast<size_t> (exports.extent (0))) {
936 (pack_pids && exports.extent (0) != 0 &&
938 "pack_pids is true, and exports.extent(0) = " <<
939 exports.extent (0) <<
" != 0, meaning that we need to pack at least "
940 "one graph entry, but export_pids.extent(0) = 0.");
942 exports.modify_device ();
943 using PackCrsGraphImpl::do_pack;
945 exports.view_device (),
946 num_packets_per_lid.view_device (),
947 export_lids.view_device (),
952template<
typename LO,
typename GO,
typename NT>
961 const Teuchos::ArrayView<const LO>&
exportLIDs,
962 const Teuchos::ArrayView<const int>&
sourcePIDs,
965 using Kokkos::HostSpace;
966 using Kokkos::MemoryUnmanaged;
969 using buffer_device_type =
typename crs_graph_type::buffer_device_type;
979 "num_packets_per_lid");
995 constexpr bool pack_pids =
true;
996 PackCrsGraphImpl::packCrsGraph
1005 using execution_space =
typename buffer_device_type::execution_space;
1006 Kokkos::deep_copy (execution_space(),
1013#define TPETRA_DETAILS_PACKCRSGRAPH_INSTANT( LO, GO, NT ) \
1015 Details::packCrsGraph<LO, GO, NT> ( \
1016 const CrsGraph<LO, GO, NT>&, \
1017 Teuchos::Array<CrsGraph<LO,GO,NT>::packet_type>&, \
1018 const Teuchos::ArrayView<size_t>&, \
1019 const Teuchos::ArrayView<const LO>&, \
1022 Details::packCrsGraphNew<LO, GO, NT> ( \
1023 const CrsGraph<LO, GO, NT>&, \
1024 const Kokkos::DualView< \
1026 CrsGraph<LO,GO,NT>::buffer_device_type>&, \
1027 const Kokkos::DualView< \
1029 CrsGraph<LO,GO,NT>::buffer_device_type>&, \
1031 CrsGraph<LO,GO,NT>::packet_type*, \
1032 CrsGraph<LO,GO,NT>::buffer_device_type>&, \
1035 CrsGraph<LO,GO,NT>::buffer_device_type>, \
1039 Details::packCrsGraphWithOwningPIDs<LO, GO, NT> ( \
1040 const CrsGraph<LO, GO, NT>&, \
1041 Kokkos::DualView<CrsGraph<LO,GO,NT>::packet_type*, CrsGraph<LO,GO,NT>::buffer_device_type>&, \
1042 const Teuchos::ArrayView<size_t>&, \
1043 const Teuchos::ArrayView<const LO>&, \
1044 const Teuchos::ArrayView<const int>&, \
Declaration of the Tpetra::CrsGraph class.
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra's behavior.
Import KokkosSparse::OrdinalTraits, a traits class for "invalid" (flag) values of integer types,...
Declaration and generic definition of traits class that tells Tpetra::CrsMatrix how to pack and unpac...
Declaration and definition of Tpetra::Details::castAwayConstDualView, an implementation detail of Tpe...
Functions that wrap Kokkos::create_mirror_view, in order to avoid deep copies when not necessary,...
Declaration and definition of Tpetra::Details::getEntryOnHost.
CountsViewType::non_const_value_type computeNumPacketsAndOffsets(const OutputOffsetsViewType &outputOffsets, const CountsViewType &counts, const InputOffsetsViewType &rowOffsets, const InputLocalRowIndicesViewType &lclRowInds, const InputLocalRowPidsViewType &lclRowPids)
Compute the number of packets and offsets for the pack procedure.
void do_pack(const LocalGraph &local_graph, const LocalMap &local_map, const Kokkos::View< Packet *, BufferDeviceType > &exports, const typename PackTraits< size_t >::input_array_type &num_packets_per_lid, const typename PackTraits< typename LocalMap::local_ordinal_type >::input_array_type &export_lids, const typename PackTraits< int >::input_array_type &source_pids, const Kokkos::View< const size_t *, BufferDeviceType > &offsets, const bool pack_pids)
Perform the pack operation for the graph.
KOKKOS_FUNCTION size_t packRow(const LocalMapType &col_map, const Kokkos::View< Packet *, BufferDeviceType > &exports, const InputLidsType &lids_in, const InputPidsType &pids_in, const size_t offset, const size_t num_ent, const bool pack_pids)
Packs a single row of the CrsGraph.
typename dist_object_type::buffer_device_type buffer_device_type
Kokkos::Device specialization for communication buffers.
Struct that holds views of the contents of a CrsMatrix.
"Local" part of Map suitable for Kokkos kernels.
LocalOrdinal local_ordinal_type
The type of local indices.
GlobalOrdinal global_ordinal_type
The type of global indices.
Compute the number of packets and offsets for the pack procedure.
int getError() const
Host function for getting the error.
Implementation details of Tpetra.
void packCrsGraph(const CrsGraph< LO, GO, NT > &sourceGraph, Teuchos::Array< typename CrsGraph< LO, GO, NT >::packet_type > &exports, const Teuchos::ArrayView< size_t > &numPacketsPerLID, const Teuchos::ArrayView< const LO > &exportLIDs, size_t &constantNumPackets)
Pack specified entries of the given local sparse graph for communication.
Impl::CreateMirrorViewFromUnmanagedHostArray< ValueType, OutputDeviceType >::output_view_type create_mirror_view_from_raw_host_array(const OutputDeviceType &, ValueType *inPtr, const size_t inSize, const bool copy=true, const char label[]="")
Variant of Kokkos::create_mirror_view that takes a raw host 1-d array as input.
void packCrsGraphNew(const CrsGraph< LO, GO, NT > &sourceGraph, const Kokkos::DualView< const LO *, typename CrsGraph< LO, GO, NT >::buffer_device_type > &exportLIDs, const Kokkos::DualView< const int *, typename CrsGraph< LO, GO, NT >::buffer_device_type > &exportPIDs, Kokkos::DualView< typename CrsGraph< LO, GO, NT >::packet_type *, typename CrsGraph< LO, GO, NT >::buffer_device_type > &exports, Kokkos::DualView< size_t *, typename CrsGraph< LO, GO, NT >::buffer_device_type > numPacketsPerLID, size_t &constantNumPackets, const bool pack_pids)
Pack specified entries of the given local sparse graph for communication, for "new" DistObject interf...
void packCrsGraphWithOwningPIDs(const CrsGraph< LO, GO, NT > &sourceGraph, Kokkos::DualView< typename CrsGraph< LO, GO, NT >::packet_type *, typename CrsGraph< LO, GO, NT >::buffer_device_type > &exports_dv, const Teuchos::ArrayView< size_t > &numPacketsPerLID, const Teuchos::ArrayView< const LO > &exportLIDs, const Teuchos::ArrayView< const int > &sourcePIDs, size_t &constantNumPackets)
Pack specified entries of the given local sparse graph for communication.
Namespace Tpetra contains the class and methods constituting the Tpetra library.
Traits class for packing / unpacking data of type T.
Kokkos::View< const value_type *, Kokkos::AnonymousSpace > input_array_type
The type of an input array of value_type.