Yet Another eXchange Tool 0.11.3
Loading...
Searching...
No Matches
xt_redist_p2p_ext.h
Go to the documentation of this file.
1
12/*
13 * Keywords:
14 * Maintainer: Jörg Behrens <behrens@dkrz.de>
15 * Moritz Hanke <hanke@dkrz.de>
16 * Thomas Jahns <jahns@dkrz.de>
17 * URL: https://dkrz-sw.gitlab-pages.dkrz.de/yaxt/
18 *
19 * Redistribution and use in source and binary forms, with or without
20 * modification, are permitted provided that the following conditions are
21 * met:
22 *
23 * Redistributions of source code must retain the above copyright notice,
24 * this list of conditions and the following disclaimer.
25 *
26 * Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in the
28 * documentation and/or other materials provided with the distribution.
29 *
30 * Neither the name of the DKRZ GmbH nor the names of its contributors
31 * may be used to endorse or promote products derived from this software
32 * without specific prior written permission.
33 *
34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
35 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
36 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
37 * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
38 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
39 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
40 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
41 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
42 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
43 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
44 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 */
46
47#define TOKEN_PASTE2_(a,b) a##b
48#define TOKEN_PASTE2(a,b) TOKEN_PASTE2_(a,b)
49#define TOKEN_PASTE3_(a,b,c) a##b##c
50#define TOKEN_PASTE3(a,b,c) TOKEN_PASTE3_(a,b,c)
51
52
53#define XT_GENERATE_EXT_DATATYPE \
54 TOKEN_PASTE3(generate_,XT_EXT_TAG,_datatype)
55#define XT_GENERATE_EXT_MSG_INFOS \
56 TOKEN_PASTE3(generate_,XT_EXT_TAG,_msg_infos)
57#define XT_REDIST_P2P_EXT_NEW \
58 TOKEN_PASTE3(xt_redist_p2p_,XT_EXT_TAG,_new)
59#define XT_REDIST_P2P_EXT_CUSTOM_NEW \
60 TOKEN_PASTE3(xt_redist_p2p_,XT_EXT_TAG,_custom_new)
61
62static MPI_Datatype
63XT_GENERATE_EXT_DATATYPE(int num_transfer_pos_ext,
64 const struct Xt_pos_ext transfer_pos_ext[],
65 int num_ext, const XT_EXT_TYPE extents[],
66 const int psum_ext_size[],
67 void **work_buf, size_t *work_buf_size,
68 struct Xt_mpi_strp_prs_params *params)
69{
70 if (num_transfer_pos_ext > 0)
71 {
72 XT_EXT_TYPE *dt_stripes;
73 size_t size_dt_stripes, num_dt_stripes = 0;
74 enum
75 {
76 dt_stripes_init_size = 8,
77 dt_stripes_init_alloc = dt_stripes_init_size * sizeof (*dt_stripes),
78 };
79 if (*work_buf_size < dt_stripes_init_alloc) {
80 dt_stripes = xrealloc(*work_buf, dt_stripes_init_alloc);
81 size_dt_stripes = dt_stripes_init_size;
82 } else {
83 dt_stripes = *work_buf;
84 size_dt_stripes = *work_buf_size / sizeof (*dt_stripes);
85 }
86 int i = 0,
87 search_start_ext
88 = pos2disp(transfer_pos_ext[0].start,
89 num_ext, psum_ext_size);
90 do
91 {
92 struct Xt_pos_ext current_pos_ext = transfer_pos_ext[i];
93 if (num_dt_stripes >= size_dt_stripes) {
94 more_stripes:
95 size_dt_stripes *= 2;
96 dt_stripes = xrealloc(dt_stripes,
97 size_dt_stripes * sizeof (*dt_stripes));
98 }
99 do {
100 /* find extent containing start position of current range */
101 search_start_ext = pos2disp2(current_pos_ext.start,
102 num_ext, psum_ext_size,
103 search_start_ext);
104 XT_EXT_TYPE base_ext = extents[search_start_ext],
105 derived_ext;
106 int pos_remaining = current_pos_ext.start
107 - psum_ext_size[search_start_ext];
108 derived_ext.start
109 = base_ext.start + pos_remaining * base_ext.stride;
110 int isign_mask_current_pos_ext_size = isign_mask(current_pos_ext.size);
112 /* find number of positions in containing extent,
113 * which precede current_pos_ext.start
114 * if (current_pos_ext.size < 0)
115 * or follow current_pos_ext.start
116 * if (current_pos_ext.size > 0) */
117 derived_ext.size = imin(abs(current_pos_ext.size),
118 (~isign_mask_current_pos_ext_size
119 & (base_ext.size - pos_remaining))
120 | (isign_mask_current_pos_ext_size
121 & (pos_remaining + 1)));
122 derived_ext.stride
123 = (~XT_EXT_STRIDE_MASK & base_ext.stride)
124 | (XT_EXT_STRIDE_MASK & -base_ext.stride);
125 dt_stripes[num_dt_stripes++] = derived_ext;
126 current_pos_ext.size
127 += (~isign_mask_current_pos_ext_size & -derived_ext.size)
128 | (isign_mask_current_pos_ext_size & derived_ext.size);
129 current_pos_ext.start += derived_ext.size;
130 } while ((abs(current_pos_ext.size) > 0)
131 & (num_dt_stripes < size_dt_stripes));
132 /* current_pos_ext hasn't been mapped completely, get more
133 * stripe memory */
134 if (abs(current_pos_ext.size) > 0)
135 goto more_stripes;
136 /* only advance current_pos_ext after it has been mapped completely */
137 } while (++i < num_transfer_pos_ext);
138
139 MPI_Datatype type
140 = XT_MPI_PARSE_STRIPE(num_dt_stripes, dt_stripes, params);
141 *work_buf = dt_stripes;
142 *work_buf_size = size_dt_stripes * sizeof (*dt_stripes);
143 return type;
144 }
145 else
146 return MPI_DATATYPE_NULL;
147}
148
149static void
151 int num_ext,
152 const XT_EXT_TYPE extents[],
153 struct Xt_redist_msg *msgs,
154 struct Xt_mpi_strp_prs_params *params)
155{
156 if (num_msgs > 0) {
157 /* partial sums of ext sizes */
158 int *restrict psum_ext_size
159 = xmalloc(((size_t)num_ext + 1) * sizeof (psum_ext_size[0]));
160 int accum = 0;
161 for (size_t i = 0; i < (size_t)num_ext; ++i) {
162 psum_ext_size[i] = accum;
163 accum += extents[i].size;
164 }
165 psum_ext_size[num_ext] = accum;
166
167 void *buf = NULL;
168 size_t buf_size = 0;
169
170 struct Xt_redist_msg *curr_msg = msgs;
171 do {
172
173 const struct Xt_pos_ext *curr_transfer_pos_ext
175 int curr_num_transfer_pos_ext
177
178 curr_msg->datatype
179 = XT_GENERATE_EXT_DATATYPE(curr_num_transfer_pos_ext,
180 curr_transfer_pos_ext,
181 num_ext, extents, psum_ext_size,
182 &buf, &buf_size,
183 params);
184 curr_msg->rank = xt_xmap_iterator_get_rank(iter);
185
186 curr_msg++;
187
188 } while (xt_xmap_iterator_next(iter));
189 free(psum_ext_size);
190 free(buf);
191 }
192}
193
196 int num_src_ext,
197 const XT_EXT_TYPE src_extents[],
198 int num_dst_ext,
199 const XT_EXT_TYPE dst_extents[],
200 MPI_Datatype datatype)
201{
202 return XT_REDIST_P2P_EXT_CUSTOM_NEW(xmap, num_src_ext, src_extents,
203 num_dst_ext, dst_extents, datatype,
205}
206
209 int num_src_ext,
210 const XT_EXT_TYPE src_extents[],
211 int num_dst_ext,
212 const XT_EXT_TYPE dst_extents[],
213 MPI_Datatype datatype,
214 Xt_config config)
215{
216 // ensure that yaxt is initialized
217 assert(xt_initialized());
218 int tag_offset;
219 MPI_Comm comm
221
222 int nrecv = xt_xmap_get_num_sources(xmap),
223 nsend = xt_xmap_get_num_destinations(xmap);
224 size_t nmsg = (size_t)nrecv + (size_t)nsend;
225 struct Xt_redist_msg *msgs = xmalloc(nmsg * sizeof (*msgs)),
226 *send_msgs = msgs, *recv_msgs = msgs + nsend;
227 Xt_xmap_iter dst_iter = xt_xmap_get_in_iterator(xmap);
228 struct Xt_mpi_strp_prs_params params;
229 xt_init_mpi_strp_prs_params(&params, datatype, comm);
230 XT_GENERATE_EXT_MSG_INFOS(nrecv, dst_iter, num_dst_ext, dst_extents,
231 recv_msgs, &params);
232 if (dst_iter) xt_xmap_iterator_delete(dst_iter);
233
234 Xt_xmap_iter src_iter = xt_xmap_get_out_iterator(xmap);
235 XT_GENERATE_EXT_MSG_INFOS(nsend, src_iter, num_src_ext, src_extents,
236 send_msgs, &params);
237 if (src_iter) xt_xmap_iterator_delete(src_iter);
239
240 struct Xt_config_ config_ = *config;
241 config_.flags |= exch_no_dt_dup;
242
244 nsend, nrecv, send_msgs, recv_msgs, comm, &config_);
245
246 free(msgs);
247 xt_mpi_comm_smart_dedup(&comm, tag_offset);
248 return result;
249}
250
251#undef TOKEN_PASTE2
252#undef TOKEN_PASTE2_
253#undef TOKEN_PASTE3
254#undef TOKEN_PASTE3_
255
256/*
257 * Local Variables:
258 * c-basic-offset: 2
259 * coding: utf-8
260 * indent-tabs-mode: nil
261 * show-trailing-whitespace: t
262 * require-trailing-newline: t
263 * End:
264 */
int MPI_Comm
Definition core.h:64
#define xrealloc(ptr, size)
Definition ppm_xfuncs.h:71
#define xmalloc(size)
Definition ppm_xfuncs.h:70
int size
Definition xt_core.h:97
int start
Definition xt_core.h:97
MPI_Datatype datatype
Definition xt_redist.h:69
static int isign_mask(int x)
static int imin(int a, int b)
struct Xt_config_ xt_default_config
Definition xt_config.c:204
@ exch_no_dt_dup
int xt_initialized(void)
MPI_Comm xt_mpi_comm_smart_dup(MPI_Comm comm, int *tag_offset)
Definition xt_mpi.c:333
void xt_mpi_comm_smart_dedup(MPI_Comm *comm, int tag_offset)
Definition xt_mpi.c:386
static void xt_destroy_mpi_strp_prs_params(struct Xt_mpi_strp_prs_params *params)
static void xt_init_mpi_strp_prs_params(struct Xt_mpi_strp_prs_params *params, MPI_Datatype old_type, MPI_Comm comm)
#define XT_EXT_STRIDE_MASK_PREP
#define XT_EXT_TYPE
static int pos2disp2(int pos, int num_ext, const int psum_ext_size[], int start_ext)
static int pos2disp(int pos, int num_ext, const int psum_ext_size[])
#define XT_MPI_PARSE_STRIPE
#define XT_EXT_STRIDE_MASK
#define XT_GENERATE_EXT_MSG_INFOS
#define XT_REDIST_P2P_EXT_CUSTOM_NEW
#define XT_REDIST_P2P_EXT_NEW
#define XT_GENERATE_EXT_DATATYPE
Xt_redist xt_redist_single_array_base_custom_new(int nsend, int nrecv, const struct Xt_redist_msg send_msgs[], const struct Xt_redist_msg recv_msgs[], MPI_Comm comm, Xt_config config)
int xt_xmap_iterator_next(Xt_xmap_iter iter)
Definition xt_xmap.c:101
Xt_xmap_iter xt_xmap_get_out_iterator(Xt_xmap xmap)
Definition xt_xmap.c:96
int xt_xmap_iterator_get_num_transfer_pos_ext(Xt_xmap_iter iter)
Definition xt_xmap.c:126
void xt_xmap_iterator_delete(Xt_xmap_iter iter)
Definition xt_xmap.c:130
int xt_xmap_get_num_destinations(Xt_xmap xmap)
Definition xt_xmap.c:61
int xt_xmap_iterator_get_rank(Xt_xmap_iter iter)
Definition xt_xmap.c:106
int xt_xmap_get_num_sources(Xt_xmap xmap)
Definition xt_xmap.c:66
const struct Xt_pos_ext * xt_xmap_iterator_get_transfer_pos_ext(Xt_xmap_iter iter)
Definition xt_xmap.c:122
Xt_xmap_iter xt_xmap_get_in_iterator(Xt_xmap xmap)
Definition xt_xmap.c:91
MPI_Comm xt_xmap_get_communicator(Xt_xmap xmap)
Definition xt_xmap.c:56