64 std::vector<std::vector<
65 std::pair<types::global_dof_index, types::global_dof_index>>>
67 std::vector<std::vector<
68 std::pair<types::global_dof_index, types::global_dof_index>>>
69 ©_indices_global_mine,
70 std::vector<std::vector<
71 std::pair<types::global_dof_index, types::global_dof_index>>>
72 & copy_indices_level_mine,
73 const bool skip_interface_dofs)
86 std::vector<DoFPair> send_data_temp;
88 const unsigned int n_levels =
90 copy_indices.resize(n_levels);
91 copy_indices_global_mine.resize(n_levels);
92 copy_indices_level_mine.resize(n_levels);
96 std::vector<types::global_dof_index> global_dof_indices(dofs_per_cell);
97 std::vector<types::global_dof_index> level_dof_indices(dofs_per_cell);
101 std::vector<bool> dof_touched(owned_dofs.
n_elements(),
false);
109 std::vector<types::global_dof_index> unrolled_copy_indices;
112 copy_indices_global_mine[
level].clear();
114 for (
const auto &level_cell :
119 (level_cell->level_subdomain_id() ==
121 level_cell->subdomain_id() ==
125 unrolled_copy_indices.resize(owned_dofs.
n_elements(),
130 level_cell->get_dof_indices(global_dof_indices);
131 level_cell->get_mg_dof_indices(level_dof_indices);
133 for (
unsigned int i = 0; i < dofs_per_cell; ++i)
137 if (skip_interface_dofs && mg_constrained_dofs !=
nullptr &&
139 level, level_dof_indices[i]))
151 owned_level_dofs.
is_element(level_dof_indices[i]);
153 if (global_mine && level_mine)
158 unrolled_copy_indices[global_index_in_set] =
159 level_dof_indices[i];
161 else if (global_mine &&
162 dof_touched[global_index_in_set] ==
false)
164 copy_indices_global_mine[
level].emplace_back(
165 global_dof_indices[i], level_dof_indices[i]);
168 send_data_temp.emplace_back(
level,
169 global_dof_indices[i],
170 level_dof_indices[i]);
171 dof_touched[global_index_in_set] =
true;
182 if (!unrolled_copy_indices.empty())
184 copy_indices[
level].clear();
188 if (copy_indices_global_mine[
level].empty())
189 copy_indices[
level].reserve(unrolled_copy_indices.size());
194 for (
unsigned int i = 0; i < unrolled_copy_indices.size(); ++i)
196 copy_indices[
level].emplace_back(
201 const ::parallel::TriangulationBase<dim, spacedim> *
tria =
202 (
dynamic_cast<const ::parallel::TriangulationBase<dim, spacedim>
205 send_data_temp.size() == 0 ||
tria !=
nullptr,
207 "We should only be sending information with a parallel Triangulation!"));
209#ifdef DEAL_II_WITH_MPI
213 const std::set<types::subdomain_id> &neighbors =
214 tria->level_ghost_owners();
215 std::map<int, std::vector<DoFPair>> send_data;
217 std::sort(send_data_temp.begin(),
218 send_data_temp.end(),
220 if (lhs.level < rhs.level)
222 if (lhs.level > rhs.level)
225 if (lhs.level_dof_index < rhs.level_dof_index)
227 if (lhs.level_dof_index > rhs.level_dof_index)
230 if (lhs.global_dof_index < rhs.global_dof_index)
235 send_data_temp.erase(
236 std::unique(send_data_temp.begin(),
237 send_data_temp.end(),
239 return (lhs.level == rhs.level) &&
240 (lhs.level_dof_index == rhs.level_dof_index) &&
241 (lhs.global_dof_index == rhs.global_dof_index);
243 send_data_temp.end());
250 std::vector<types::global_dof_index> level_dof_indices;
251 std::vector<types::global_dof_index> global_dof_indices;
252 for (
const auto &dofpair : send_data_temp)
253 if (dofpair.level ==
level)
255 level_dof_indices.push_back(dofpair.level_dof_index);
256 global_dof_indices.push_back(dofpair.global_dof_index);
261 level_dof_indices.end());
266 const auto index_owner =
271 AssertThrow(level_dof_indices.size() == index_owner.size(),
274 for (
unsigned int i = 0; i < index_owner.size(); ++i)
275 send_data[index_owner[i]].emplace_back(
level,
276 global_dof_indices[i],
277 level_dof_indices[i]);
290 std::vector<MPI_Request> requests;
292 for (
const auto dest : neighbors)
294 requests.push_back(MPI_Request());
295 std::vector<DoFPair> &data = send_data[dest];
298 MPI_Isend(data.data(),
299 data.size() *
sizeof(
decltype(*data.data())),
304 &*requests.rbegin());
312 std::vector<DoFPair> receive_buffer;
313 for (
unsigned int counter = 0; counter < neighbors.size();
317 int ierr = MPI_Probe(MPI_ANY_SOURCE,
323 ierr = MPI_Get_count(&status, MPI_BYTE, &len);
328 ierr = MPI_Recv(
nullptr,
339 int count = len /
sizeof(
DoFPair);
342 receive_buffer.resize(count);
344 void *ptr = receive_buffer.data();
354 for (
const auto &dof_pair : receive_buffer)
356 copy_indices_level_mine[dof_pair.level].emplace_back(
357 dof_pair.global_dof_index, dof_pair.level_dof_index);
363 if (requests.size() > 0)
365 const int ierr = MPI_Waitall(requests.size(),
367 MPI_STATUSES_IGNORE);
385 std::less<std::pair<types::global_dof_index, types::global_dof_index>>
387 for (
auto &level_indices : copy_indices_level_mine)
388 std::sort(level_indices.begin(), level_indices.end(), compare);
389 for (
auto &level_indices : copy_indices_global_mine)
390 std::sort(level_indices.begin(), level_indices.end(), compare);
400 std::vector<types::global_dof_index> &ghosted_level_dofs,
401 const std::shared_ptr<const Utilities::MPI::Partitioner>
402 & external_partitioner,
404 std::shared_ptr<const Utilities::MPI::Partitioner> &target_partitioner,
407 std::sort(ghosted_level_dofs.begin(), ghosted_level_dofs.end());
409 ghosted_dofs.
add_indices(ghosted_level_dofs.begin(),
410 std::unique(ghosted_level_dofs.begin(),
411 ghosted_level_dofs.end()));
415 if (target_partitioner.get() !=
nullptr &&
416 target_partitioner->size() == locally_owned.
size())
418 ghosted_dofs.
add_indices(target_partitioner->ghost_indices());
423 const int ghosts_locally_contained =
424 (external_partitioner.get() !=
nullptr &&
425 (external_partitioner->ghost_indices() & ghosted_dofs) ==
429 if (external_partitioner.get() !=
nullptr &&
435 if (target_partitioner.get() !=
nullptr &&
436 target_partitioner->size() == locally_owned.
size())
437 for (
unsigned int i = 0; i < copy_indices_global_mine.n_cols(); ++i)
438 copy_indices_global_mine(1, i) =
439 external_partitioner->global_to_local(
440 target_partitioner->local_to_global(
441 copy_indices_global_mine(1, i)));
442 target_partitioner = external_partitioner;
446 if (target_partitioner.get() !=
nullptr &&
447 target_partitioner->size() == locally_owned.
size())
448 for (
unsigned int i = 0; i < copy_indices_global_mine.n_cols(); ++i)
449 copy_indices_global_mine(1, i) =
452 target_partitioner->local_to_global(
453 copy_indices_global_mine(1, i)));
455 std::make_shared<Utilities::MPI::Partitioner>(locally_owned,
519 const unsigned int child,
520 const unsigned int fe_shift_1d,
521 const unsigned int fe_degree,
522 const std::vector<unsigned int> & lexicographic_numbering,
523 const std::vector<types::global_dof_index> &local_dof_indices,
526 const unsigned int n_child_dofs_1d = fe_degree + 1 + fe_shift_1d;
527 const unsigned int shift =
528 compute_shift_within_children<dim>(child, fe_shift_1d, fe_degree);
529 const unsigned int n_components =
530 local_dof_indices.size() / Utilities::fixed_power<dim>(fe_degree + 1);
532 const unsigned int n_scalar_cell_dofs =
533 Utilities::fixed_power<dim>(n_child_dofs_1d);
534 for (
unsigned int c = 0, m = 0; c < n_components; ++c)
535 for (
unsigned int k = 0; k < (dim > 2 ? (fe_degree + 1) : 1); ++k)
536 for (
unsigned int j = 0; j < (dim > 1 ? (fe_degree + 1) : 1); ++j)
537 for (
unsigned int i = 0; i < (fe_degree + 1); ++i, ++m)
539 const unsigned int index =
540 c * n_scalar_cell_dofs +
541 k * n_child_dofs_1d * n_child_dofs_1d + j * n_child_dofs_1d +
545 local_dof_indices[lexicographic_numbering[m]],
547 indices[
index] = local_dof_indices[lexicographic_numbering[m]];
622 const std::vector<std::shared_ptr<const Utilities::MPI::Partitioner>>
623 & external_partitioners,
625 std::vector<std::vector<unsigned int>> &level_dof_indices,
626 std::vector<std::vector<std::pair<unsigned int, unsigned int>>>
627 & parent_child_connect,
628 std::vector<unsigned int> &n_owned_level_cells,
629 std::vector<std::vector<std::vector<unsigned short>>> &dirichlet_indices,
630 std::vector<std::vector<Number>> & weights_on_refined,
632 MGLevelObject<std::shared_ptr<const Utilities::MPI::Partitioner>>
633 &target_partitioners)
635 level_dof_indices.clear();
636 parent_child_connect.clear();
637 n_owned_level_cells.clear();
638 dirichlet_indices.clear();
639 weights_on_refined.clear();
653 const std::size_t template_starts = fe_name.find_first_of(
'<');
654 Assert(fe_name[template_starts + 1] ==
655 (dim == 1 ?
'1' : (dim == 2 ?
'2' :
'3')),
657 fe_name[template_starts + 1] =
'1';
659 const std::unique_ptr<FiniteElement<1>> fe(
660 FETools::get_fe_by_name<1, 1>(fe_name));
667 level_dof_indices.resize(n_levels);
668 parent_child_connect.resize(n_levels - 1);
669 n_owned_level_cells.resize(n_levels - 1);
670 std::vector<std::vector<unsigned int>> coarse_level_indices(n_levels - 1);
671 for (
unsigned int level = 0;
676 std::vector<types::global_dof_index> local_dof_indices(
678 dirichlet_indices.resize(n_levels - 1);
681 Assert(external_partitioners.empty() ||
682 external_partitioners.size() == n_levels,
687 unsigned int counter = 0;
688 std::vector<types::global_dof_index> global_level_dof_indices;
689 std::vector<types::global_dof_index> global_level_dof_indices_remote;
690 std::vector<types::global_dof_index> ghosted_level_dofs;
691 std::vector<types::global_dof_index> global_level_dof_indices_l0;
692 std::vector<types::global_dof_index> ghosted_level_dofs_l0;
697 for (cell = dof_handler.
begin(
level - 1); cell != endc; ++cell)
701 if (!cell->has_children())
712 const bool cell_is_remote = !consider_cell;
713 for (
unsigned int c = 0;
714 c < GeometryInfo<dim>::max_children_per_cell;
716 if (cell->child(c)->level_subdomain_id() ==
719 consider_cell =
true;
734 std::vector<types::global_dof_index> &next_indices =
735 cell_is_remote ? global_level_dof_indices_remote :
736 global_level_dof_indices;
737 const std::size_t start_index = next_indices.size();
740 for (
unsigned int c = 0;
741 c < GeometryInfo<dim>::max_children_per_cell;
744 if (cell_is_remote && cell->child(c)->level_subdomain_id() !=
747 cell->child(c)->get_mg_dof_indices(local_dof_indices);
755 for (
const auto local_dof_index : local_dof_indices)
756 if (!owned_level_dofs.
is_element(local_dof_index))
757 ghosted_level_dofs.push_back(local_dof_index);
759 add_child_indices<dim>(c,
760 fe->n_dofs_per_cell() -
761 fe->n_dofs_per_vertex(),
765 &next_indices[start_index]);
768 if (cell->child(c)->has_children() &&
771 cell->child(c)->level_subdomain_id() ==
774 const unsigned int child_index =
775 coarse_level_indices[
level][cell->child(c)->index()];
777 parent_child_connect[
level].size());
778 unsigned int parent_index = counter;
789 parent_child_connect[
level][child_index] =
790 std::make_pair(parent_index, c);
792 static_cast<unsigned short>(-1));
796 if (mg_constrained_dofs !=
nullptr)
797 for (
unsigned int i = 0;
798 i < dof_handler.
get_fe().n_dofs_per_cell();
804 dirichlet_indices[
level][child_index].push_back(i);
810 coarse_level_indices[
level - 1].size());
811 coarse_level_indices[
level - 1][cell->index()] = counter++;
818 if (
level == 1 && !cell_is_remote)
820 cell->get_mg_dof_indices(local_dof_indices);
826 const IndexSet &owned_level_dofs_l0 =
828 for (
const auto local_dof_index : local_dof_indices)
829 if (!owned_level_dofs_l0.
is_element(local_dof_index))
830 ghosted_level_dofs_l0.push_back(local_dof_index);
832 const std::size_t start_index =
833 global_level_dof_indices_l0.size();
834 global_level_dof_indices_l0.resize(
837 add_child_indices<dim>(
839 fe->n_dofs_per_cell() - fe->n_dofs_per_vertex(),
843 &global_level_dof_indices_l0[start_index]);
845 dirichlet_indices[0].emplace_back();
846 if (mg_constrained_dofs !=
nullptr)
847 for (
unsigned int i = 0;
848 i < dof_handler.
get_fe().n_dofs_per_cell();
852 local_dof_indices[elem_info
854 dirichlet_indices[0].back().push_back(i);
862 global_level_dof_indices.size());
863 n_owned_level_cells[
level - 1] = counter;
864 dirichlet_indices[
level - 1].resize(counter);
865 parent_child_connect[
level - 1].resize(
873 if (
level < n_levels - 1)
874 for (std::vector<std::pair<unsigned int, unsigned int>>::iterator
875 i = parent_child_connect[
level].begin();
876 i != parent_child_connect[
level].end();
892 external_partitioners.empty() ?
894 external_partitioners[
level],
896 target_partitioners[
level],
897 copy_indices_global_mine[
level]);
900 global_level_dof_indices,
901 global_level_dof_indices_remote,
902 level_dof_indices[
level]);
906 for (
unsigned int i = 0; i < parent_child_connect[0].size(); ++i)
907 parent_child_connect[0][i] = std::make_pair(i, 0U);
910 ghosted_level_dofs_l0,
911 external_partitioners.empty() ?
913 external_partitioners[0],
915 target_partitioners[0],
916 copy_indices_global_mine[0]);
919 *target_partitioners[0],
920 global_level_dof_indices_l0,
921 std::vector<types::global_dof_index>(),
922 level_dof_indices[0]);
928 const unsigned int n_child_dofs_1d =
929 fe->degree + 1 + fe->n_dofs_per_cell() - fe->n_dofs_per_vertex();
933 weights_on_refined.resize(n_levels - 1);
937 target_partitioners[
level]);
938 for (
unsigned int c = 0; c < n_owned_level_cells[
level - 1]; ++c)
946 std::vector<unsigned int> degree_to_3(n_child_dofs_1d);
948 for (
unsigned int i = 1; i < n_child_dofs_1d - 1; ++i)
950 degree_to_3.back() = 2;
954 weights_on_refined[
level - 1].resize(n_owned_level_cells[
level - 1] *
955 Utilities::fixed_power<dim>(3));
956 for (
unsigned int c = 0; c < n_owned_level_cells[
level - 1]; ++c)
957 for (
unsigned int k = 0, m = 0; k < (dim > 2 ? n_child_dofs_1d : 1);
959 for (
unsigned int j = 0; j < (dim > 1 ? n_child_dofs_1d : 1); ++j)
961 unsigned int shift = 9 * degree_to_3[k] + 3 * degree_to_3[j];
962 for (
unsigned int i = 0; i < n_child_dofs_1d; ++i, ++m)
963 weights_on_refined[
level -
964 1][c * Utilities::fixed_power<dim>(3) +
965 shift + degree_to_3[i]] =
968 level_dof_indices[
level]