DOLFIN-X
DOLFIN-X C++ interface
IndexMap.h
1 // Copyright (C) 2015-2019 Chris Richardson, Garth N. Wells and Igor Baratta
2 //
3 // This file is part of DOLFINX (https://www.fenicsproject.org)
4 //
5 // SPDX-License-Identifier: LGPL-3.0-or-later
6 
7 #pragma once
8 
9 #include <array>
10 #include <cstdint>
11 #include <dolfinx/common/MPI.h>
12 #include <map>
13 #include <tuple>
14 #include <utility>
15 #include <vector>
16 
17 namespace dolfinx::common
18 {
19 // Forward declaration
20 class IndexMap;
21 
31 std::tuple<std::int64_t, std::vector<std::int32_t>,
32  std::vector<std::vector<std::int64_t>>,
33  std::vector<std::vector<int>>>
35  const std::vector<
36  std::pair<std::reference_wrapper<const common::IndexMap>, int>>& maps);
37 
44 
45 class IndexMap
46 {
47 public:
49  enum class Mode
50  {
51  insert,
52  add
53  };
54 
56  enum class Direction
57  {
58  reverse, // Ghost to owner
59  forward, // Owner to ghost
60  symmetric // Symmetric. NOTE: To be removed
61  };
62 
70  IndexMap(MPI_Comm comm, std::int32_t local_size);
71 
84  IndexMap(MPI_Comm mpi_comm, std::int32_t local_size,
85  const std::vector<int>& dest_ranks,
86  const std::vector<std::int64_t>& ghosts,
87  const std::vector<int>& src_ranks);
88 
89  // Copy constructor
90  IndexMap(const IndexMap& map) = delete;
91 
93  IndexMap(IndexMap&& map) = default;
94 
96  ~IndexMap() = default;
97 
99  IndexMap& operator=(IndexMap&& map) = default;
100 
101  // Copy assignment
102  IndexMap& operator=(const IndexMap& map) = delete;
103 
105  std::array<std::int64_t, 2> local_range() const noexcept;
106 
108  std::int32_t num_ghosts() const noexcept;
109 
111  std::int32_t size_local() const noexcept;
112 
114  std::int64_t size_global() const noexcept;
115 
118  const std::vector<std::int64_t>& ghosts() const noexcept;
119 
125  MPI_Comm comm(Direction dir = Direction::symmetric) const;
126 
131  void local_to_global(const std::int32_t* local, int n,
132  std::int64_t* global) const;
133 
138  std::vector<std::int32_t>
139  global_to_local(const std::vector<std::int64_t>& indices) const;
140 
144  std::vector<std::int64_t> global_indices() const;
145 
150  const std::vector<std::int32_t>& shared_indices() const noexcept;
151 
153  std::vector<int> ghost_owner_rank() const;
154 
161  std::map<std::int32_t, std::set<int>> compute_shared_indices() const;
162 
173  void scatter_fwd(const std::vector<std::int64_t>& local_data,
174  std::vector<std::int64_t>& remote_data, int n) const;
175 
186  void scatter_fwd(const std::vector<std::int32_t>& local_data,
187  std::vector<std::int32_t>& remote_data, int n) const;
188 
199  std::vector<std::int64_t>
200  scatter_fwd(const std::vector<std::int64_t>& local_data, int n) const;
201 
211  std::vector<std::int32_t>
212  scatter_fwd(const std::vector<std::int32_t>& local_data, int n) const;
213 
223  void scatter_rev(std::vector<std::int64_t>& local_data,
224  const std::vector<std::int64_t>& remote_data, int n,
225  IndexMap::Mode op) const;
226 
236  void scatter_rev(std::vector<std::int32_t>& local_data,
237  const std::vector<std::int32_t>& remote_data, int n,
238  IndexMap::Mode op) const;
239 
240 private:
241  // Range of indices (global) owned by this process
242  std::array<std::int64_t, 2> _local_range;
243 
244  // Number indices across communicator
245  std::int64_t _size_global;
246 
247  // MPI neighborhood communicators
248 
249  // Communicator where the source ranks own the indices in the callers
250  // halo, and the destination ranks 'ghost' indices owned by the
251  // caller. I.e.,
252  // - in-edges (src) are from ranks that own my ghosts
253  // - out-edges (dest) go to ranks that 'ghost' my owned indices
254  dolfinx::MPI::Comm _comm_owner_to_ghost;
255 
256  // Communicator where the source ranks have ghost indices that are
257  // owned by the caller, and the destination ranks are the owners of
258  // indices in the callers halo region. I.e.,
259  // - in-edges (src) are from ranks that 'ghost' my owned indicies
260  // - out-edges (dest) are to the owning ranks of my ghost indices
261  dolfinx::MPI::Comm _comm_ghost_to_owner;
262 
263  // TODO: remove
264  dolfinx::MPI::Comm _comm_symmetric;
265 
266  // Local-to-global map for ghost indices
267  std::vector<std::int64_t> _ghosts;
268 
269  // Owning neighborhood rank (out edge) on '_comm_owner_to_ghost'
270  // communicator for each ghost index
271  std::vector<std::int32_t> _ghost_owners;
272 
273  // TODO: replace _shared_disp and _shared_disp by an AdjacencyList
274 
275  // TODO: _shared_indices are received on _comm_ghost_to_owner, and
276  // _shared_indices is the recv_disp on _comm_ghost_to_owner. Check for
277  // corect use on _comm_owner_to_ghost. Can guarantee that
278  // _comm_owner_to_ghost and _comm_ghost_to_owner are the transpose of
279  // each other?
280 
281  // Owned local indices that are in the halo (ghost) region on other
282  // ranks
283  std::vector<std::int32_t> _shared_indices;
284 
285  // FIXME: explain better the ranks
286  // Displacement vector for _shared_indices. _shared_indices[i] is the
287  // starting postion in _shared_indices for data that is ghosted on
288  // rank i, where i is the ith outgoing edge on _comm_owner_to_ghost.
289  std::vector<std::int32_t> _shared_disp;
290 
291  template <typename T>
292  void scatter_fwd_impl(const std::vector<T>& local_data,
293  std::vector<T>& remote_data, int n) const;
294  template <typename T>
295  void scatter_rev_impl(std::vector<T>& local_data,
296  const std::vector<T>& remote_data, int n,
297  Mode op) const;
298 };
299 
300 } // namespace dolfinx::common
This class provides utility functions for easy communication with MPI and handles cases when DOLFINX ...
Definition: MPI.h:31
This class represents the distribution index arrays across processes. An index array is a contiguous ...
Definition: IndexMap.h:46
std::map< std::int32_t, std::set< int > > compute_shared_indices() const
Definition: IndexMap.cpp:543
MPI_Comm comm(Direction dir=Direction::symmetric) const
Return a MPI communicator with attached distributed graph topology information.
Definition: IndexMap.cpp:528
std::vector< int > ghost_owner_rank() const
Owner rank (on global communicator) of each ghost entry.
Definition: IndexMap.cpp:511
~IndexMap()=default
Destructor.
std::array< std::int64_t, 2 > local_range() const noexcept
Range of indices (global) owned by this process.
Definition: IndexMap.cpp:429
Mode
Mode for reverse scatter operation.
Definition: IndexMap.h:50
IndexMap(IndexMap &&map)=default
Move constructor.
std::vector< std::int32_t > global_to_local(const std::vector< std::int64_t > &indices) const
Compute local indices for array of global indices.
Definition: IndexMap.cpp:478
void scatter_fwd(const std::vector< std::int64_t > &local_data, std::vector< std::int64_t > &remote_data, int n) const
Send n values for each index that is owned to processes that have the index as a ghost....
Definition: IndexMap.cpp:659
Direction
Edge directions of neighborhood communicator.
Definition: IndexMap.h:57
std::int64_t size_global() const noexcept
Number indices across communicator.
Definition: IndexMap.cpp:441
std::vector< std::int64_t > global_indices() const
Global indices.
Definition: IndexMap.cpp:464
IndexMap(MPI_Comm comm, std::int32_t local_size)
Create an non-overlapping index map with local_size owned on this process.
Definition: IndexMap.cpp:306
std::int32_t num_ghosts() const noexcept
Number of ghost indices on this process.
Definition: IndexMap.cpp:434
void local_to_global(const std::int32_t *local, int n, std::int64_t *global) const
Compute global indices for array of local indices.
Definition: IndexMap.cpp:448
void scatter_rev(std::vector< std::int64_t > &local_data, const std::vector< std::int64_t > &remote_data, int n, IndexMap::Mode op) const
Send n values for each ghost index to owning to the process.
Definition: IndexMap.cpp:687
IndexMap & operator=(IndexMap &&map)=default
Move assignment.
const std::vector< std::int64_t > & ghosts() const noexcept
Local-to-global map for ghosts (local indexing beyond end of local range)
Definition: IndexMap.cpp:443
const std::vector< std::int32_t > & shared_indices() const noexcept
Definition: IndexMap.cpp:506
std::int32_t size_local() const noexcept
Number of indices owned by on this process.
Definition: IndexMap.cpp:436
Miscellaneous classes, functions and types.
std::tuple< std::int64_t, std::vector< std::int32_t >, std::vector< std::vector< std::int64_t > >, std::vector< std::vector< int > > > stack_index_maps(const std::vector< std::pair< std::reference_wrapper< const common::IndexMap >, int >> &maps)
Compute layout data and ghost indices for a stacked (concatenated) index map, i.e....
Definition: IndexMap.cpp:189