Yet Another eXchange Tool 0.11.4
Loading...
Searching...
No Matches
xt_exchanger_irecv_isend.c
Go to the documentation of this file.
1
12/*
13 * Keywords:
14 * Maintainer: Jörg Behrens <behrens@dkrz.de>
15 * Moritz Hanke <hanke@dkrz.de>
16 * Thomas Jahns <jahns@dkrz.de>
17 * URL: https://dkrz-sw.gitlab-pages.dkrz.de/yaxt/
18 *
19 * Redistribution and use in source and binary forms, with or without
20 * modification, are permitted provided that the following conditions are
21 * met:
22 *
23 * Redistributions of source code must retain the above copyright notice,
24 * this list of conditions and the following disclaimer.
25 *
26 * Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in the
28 * documentation and/or other materials provided with the distribution.
29 *
30 * Neither the name of the DKRZ GmbH nor the names of its contributors
31 * may be used to endorse or promote products derived from this software
32 * without specific prior written permission.
33 *
34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
35 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
36 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
37 * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
38 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
39 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
40 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
41 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
42 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
43 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
44 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 */
46#ifdef HAVE_CONFIG_H
47#include <config.h>
48#endif
49
50#include <mpi.h>
51#ifdef _OPENMP
52#include <omp.h>
53#endif
54
55#include "core/ppm_xfuncs.h"
56#include "xt_config_internal.h"
57#include "xt/xt_mpi.h"
58#include "xt/xt_request_msgs.h"
60#include "xt_mpi_internal.h"
61#include "xt_redist_internal.h"
64
65/* unfortunately GCC 11 cannot handle the literal constants used for
66 * MPI_STATUSES_IGNORE by MPICH */
67#if __GNUC__ >= 11 && __GNUC__ <= 13 && defined MPICH
68#pragma GCC diagnostic push
69#pragma GCC diagnostic ignored "-Wstringop-overread"
70#pragma GCC diagnostic ignored "-Wstringop-overflow"
71#endif
72
73static void
74redist_msgs_to_req(const void *src_data, void *dst_data,
75 int nsend, int nrecv,
76 const struct Xt_redist_msg *send_msgs,
77 const struct Xt_redist_msg *recv_msgs,
78 int tag_offset, MPI_Comm comm,
79 MPI_Request *requests)
80{
81 for (int i = 0; i < nrecv; ++i)
82 xt_mpi_call(MPI_Irecv(dst_data, 1, recv_msgs[i].datatype,
83 recv_msgs[i].rank,
84 tag_offset + xt_mpi_tag_exchange_msg, comm,
85 requests+i), comm);
86
87 for (int i = 0; i < nsend; ++i)
88 xt_mpi_call(MPI_Isend(CAST_MPI_SEND_BUF(src_data), 1, send_msgs[i].datatype,
89 send_msgs[i].rank,
90 tag_offset + xt_mpi_tag_exchange_msg, comm,
91 requests+nrecv+i), comm);
92}
93
94static void
95xt_exchanger_irecv_isend_s_exchange(const void *src_data, void *dst_data,
96 int nsend, int nrecv,
97 const struct Xt_redist_msg *send_msgs,
98 const struct Xt_redist_msg *recv_msgs,
99 int tag_offset, MPI_Comm comm) {
100 MPI_Request *requests
101 = xmalloc((size_t)(nrecv + nsend) * sizeof (*requests));
102
103 redist_msgs_to_req(src_data, dst_data, nsend, nrecv,
104 send_msgs, recv_msgs, tag_offset,
105 comm, requests);
106 xt_mpi_call(MPI_Waitall(nrecv + nsend, requests, MPI_STATUSES_IGNORE), comm);
107 free(requests);
108}
109
110#ifdef _OPENMP
111static void
112xt_exchanger_irecv_isend_a_exchange_mt(const void *src_data, void *dst_data,
113 int nsend, int nrecv,
114 const struct Xt_redist_msg *send_msgs,
115 const struct Xt_redist_msg *recv_msgs,
116 int tag_offset, MPI_Comm comm,
117 Xt_exchanger_omp_share shared_req) {
118 MPI_Request *requests
120 int num_threads = omp_get_num_threads(),
121 tid = omp_get_thread_num();
122 int start_send = (nsend * tid) / num_threads,
123 nsend_ = (nsend * (tid+1)) / num_threads - start_send,
124 start_recv = (nrecv * tid) / num_threads,
125 nrecv_ = (nrecv * (tid+1)) / num_threads - start_recv;
126
127 redist_msgs_to_req(src_data, dst_data,
128 nsend_, nrecv_,
129 send_msgs+start_send, recv_msgs+start_recv, tag_offset,
130 comm, requests+start_recv+start_send);
131}
132
133static void
134xt_exchanger_irecv_isend_s_exchange_mt(const void *src_data, void *dst_data,
135 int nsend, int nrecv,
136 const struct Xt_redist_msg *send_msgs,
137 const struct Xt_redist_msg *recv_msgs,
138 int tag_offset, MPI_Comm comm,
139 Xt_exchanger_omp_share shared_req)
140{
141 int num_threads = omp_get_num_threads(),
142 tid = omp_get_thread_num();
143 xt_exchanger_irecv_isend_a_exchange_mt(src_data, dst_data,
144 nsend, nrecv,
145 send_msgs,
146 recv_msgs,
147 tag_offset, comm,
148 shared_req);
149 MPI_Request *requests
151 int start_send = (nsend * tid) / num_threads,
152 nsend_ = (nsend * (tid+1)) / num_threads - start_send,
153 start_recv = (nrecv * tid) / num_threads,
154 nrecv_ = (nrecv * (tid+1)) / num_threads - start_recv,
155 start_req = start_send+start_recv,
156 nreq_ = nrecv_+nsend_;
157 xt_mpi_call(MPI_Waitall(nreq_, requests+start_req,
158 MPI_STATUSES_IGNORE), comm);
159}
160
161static void
162xt_exchanger_irecv_isend_s_exchange_omp(const void *src_data, void *dst_data,
163 int nsend, int nrecv,
164 const struct Xt_redist_msg *send_msgs,
165 const struct Xt_redist_msg *recv_msgs,
166 int tag_offset, MPI_Comm comm)
167{
168 Xt_exchanger_omp_share shared_req
171#pragma omp parallel
172 xt_exchanger_irecv_isend_s_exchange_mt(src_data, dst_data,
173 nsend, nrecv,
174 send_msgs,
175 recv_msgs,
176 tag_offset, comm,
177 shared_req);
178 free(shared_req);
179}
180
181
182#endif
183
184static void
185xt_exchanger_irecv_isend_a_exchange(const void *src_data, void *dst_data,
186 int nsend, int nrecv,
187 const struct Xt_redist_msg * send_msgs,
188 const struct Xt_redist_msg * recv_msgs,
189 int tag_offset, MPI_Comm comm,
190 Xt_request *request) {
191
192 struct Xt_config_ conf = xt_default_config;
194 Xt_request requests_ = *request
195 = xt_request_msgs_alloc(nrecv + nsend, comm, &conf);
196
197 MPI_Request *requests
198 = xt_request_msgs_get_req_ptr(requests_);
199 redist_msgs_to_req(src_data, dst_data, nsend, nrecv,
200 send_msgs, recv_msgs, tag_offset, comm, requests);
201}
202
203#ifdef _OPENMP
204static void
205xt_exchanger_irecv_isend_a_exchange_omp(const void *src_data, void *dst_data,
206 int nsend, int nrecv,
207 const struct Xt_redist_msg * send_msgs,
208 const struct Xt_redist_msg * recv_msgs,
209 int tag_offset, MPI_Comm comm,
210 Xt_request *request) {
211
212 struct Xt_config_ conf = xt_default_config;
214 Xt_exchanger_omp_share shared_req
216 *request = xt_request_msgs_alloc(nrecv + nsend, comm, &conf));
217#pragma omp parallel
218 xt_exchanger_irecv_isend_a_exchange_mt(src_data, dst_data,
219 nsend, nrecv,
220 send_msgs, recv_msgs,
221 tag_offset, comm,
222 shared_req);
223}
224#endif
225
228 int nsend, int nrecv,
229 const struct Xt_redist_msg *send_msgs,
230 const struct Xt_redist_msg *recv_msgs,
231 MPI_Comm comm)
232{
233 (void)send_msgs;
234 (void)recv_msgs;
235 struct Xt_config_ conf = xt_default_config;
237 int nmsg = nsend + nrecv;
238 return (Xt_exchanger_omp_share)xt_request_msgs_alloc(nmsg, comm, &conf);
239}
240
241
243xt_exchanger_irecv_isend_new(int nsend, int nrecv,
244 const struct Xt_redist_msg *send_msgs,
245 const struct Xt_redist_msg *recv_msgs,
246 MPI_Comm comm, int tag_offset,
247 Xt_config config) {
248
253 static const xt_simple_s_exchange_func
254 s_exch_by_mthread_mode[] = {
256#ifdef _OPENMP
257 xt_exchanger_irecv_isend_s_exchange_omp,
258#else
260#endif
261 };
262 static const xt_simple_a_exchange_func
263 a_exch_by_mthread_mode[] = {
265#ifdef _OPENMP
266 xt_exchanger_irecv_isend_a_exchange_omp,
267#else
269#endif
270 };
271 int mthread_mode = xt_config_get_redist_mthread_mode(config);
272 return xt_exchanger_simple_base_new(nsend, nrecv, send_msgs, recv_msgs,
273 comm, tag_offset,
274 s_exch_by_mthread_mode[mthread_mode],
275 a_exch_by_mthread_mode[mthread_mode],
277 config);
278}
279
280
281/*
282 * Local Variables:
283 * c-basic-offset: 2
284 * coding: utf-8
285 * indent-tabs-mode: nil
286 * show-trailing-whitespace: t
287 * require-trailing-newline: t
288 * End:
289 */
int MPI_Comm
Definition core.h:64
add versions of standard API functions not returning on error
#define xmalloc(size)
Definition ppm_xfuncs.h:70
struct Xt_config_ xt_default_config
Definition xt_config.c:204
void xt_config_set_redist_mthread_mode(Xt_config config, int mode)
Definition xt_config.c:347
@ XT_MT_OPENMP
Definition xt_config.h:142
@ XT_MT_NONE
Definition xt_config.h:140
int xt_config_get_redist_mthread_mode(Xt_config config)
Definition xt_config.c:340
implementation of configuration object
struct Xt_exchanger_omp_share_ * Xt_exchanger_omp_share
static Xt_exchanger_omp_share xt_exchanger_irecv_isend_create_omp_share(int nsend, int nrecv, const struct Xt_redist_msg *send_msgs, const struct Xt_redist_msg *recv_msgs, MPI_Comm comm)
static void redist_msgs_to_req(const void *src_data, void *dst_data, int nsend, int nrecv, const struct Xt_redist_msg *send_msgs, const struct Xt_redist_msg *recv_msgs, int tag_offset, MPI_Comm comm, MPI_Request *requests)
static void xt_exchanger_irecv_isend_s_exchange(const void *src_data, void *dst_data, int nsend, int nrecv, const struct Xt_redist_msg *send_msgs, const struct Xt_redist_msg *recv_msgs, int tag_offset, MPI_Comm comm)
static void xt_exchanger_irecv_isend_a_exchange(const void *src_data, void *dst_data, int nsend, int nrecv, const struct Xt_redist_msg *send_msgs, const struct Xt_redist_msg *recv_msgs, int tag_offset, MPI_Comm comm, Xt_request *request)
Xt_exchanger xt_exchanger_irecv_isend_new(int nsend, int nrecv, const struct Xt_redist_msg *send_msgs, const struct Xt_redist_msg *recv_msgs, MPI_Comm comm, int tag_offset, Xt_config config)
Xt_exchanger xt_exchanger_simple_base_new(int nsend, int nrecv, const struct Xt_redist_msg *send_msgs, const struct Xt_redist_msg *recv_msgs, MPI_Comm comm, int tag_offset, xt_simple_s_exchange_func s_func, xt_simple_a_exchange_func a_func, xt_simple_create_omp_share_func create_omp_share_func, Xt_config config)
void(* xt_simple_s_exchange_func)(const void *src_data, void *dst_data, int nsend, int nrecv, const struct Xt_redist_msg *send_msgs, const struct Xt_redist_msg *recv_msgs, int tag_offset, MPI_Comm comm)
void(* xt_simple_a_exchange_func)(const void *src_data, void *dst_data, int nsend, int nrecv, const struct Xt_redist_msg *send_msgs, const struct Xt_redist_msg *recv_msgs, int tag_offset, MPI_Comm comm, Xt_request *request)
utility routines for MPI
#define xt_mpi_call(call, comm)
Definition xt_mpi.h:68
@ xt_mpi_tag_exchange_msg
redistribution of data, non-public declarations
MPI_Request * xt_request_msgs_get_req_ptr(Xt_request request)
Xt_request xt_request_msgs_alloc(int n, MPI_Comm comm, Xt_config config)
internal interfaces for xt_request_msgs