Actual source code: sycldevice.sycl.cxx
1: #include "sycldevice.hpp"
2: #include <limits> // for std::numeric_limits
3: #include <csetjmp> // for MPI sycl device awareness
4: #include <csignal> // SIGSEGV
5: #include <vector>
6: #include <CL/sycl.hpp>
8: namespace Petsc
9: {
11: namespace device
12: {
14: namespace sycl
15: {
17: // definition for static
18: std::array<Device::DeviceInternal *, PETSC_DEVICE_MAX_DEVICES> Device::devices_array_ = {};
19: Device::DeviceInternal **Device::devices_ = &Device::devices_array_[1];
20: int Device::defaultDevice_ = PETSC_SYCL_DEVICE_NONE;
21: bool Device::initialized_ = false;
23: static std::jmp_buf MPISyclAwareJumpBuffer;
24: static bool MPISyclAwareJumpBufferSet;
26: // internal "impls" class for SyclDevice. Each instance represents a single sycl device
27: class PETSC_NODISCARD Device::DeviceInternal {
28: const int id_; // -1 for the host device; 0 and up for gpu devices
29: bool devInitialized_;
30: const ::sycl::device syclDevice_;
32: public:
33: // default constructor
34: DeviceInternal(int id) noexcept : id_(id), devInitialized_(false), syclDevice_(chooseSYCLDevice_(id)) { }
35: int id() const { return id_; }
36: bool initialized() const { return devInitialized_; }
38: PetscErrorCode initialize() noexcept
39: {
40: PetscFunctionBegin;
41: if (initialized()) PetscFunctionReturn(PETSC_SUCCESS);
42: if (syclDevice_.is_gpu() && use_gpu_aware_mpi) {
43: if (!isMPISyclAware_()) {
44: PetscCall((*PetscErrorPrintf)("PETSc is configured with sycl support, but your MPI is not aware of sycl GPU devices. For better performance, please use a sycl GPU-aware MPI.\n"));
45: PetscCall((*PetscErrorPrintf)("If you do not care, add option -use_gpu_aware_mpi 0. To not see the message again, add the option to your .petscrc, OR add it to the env var PETSC_OPTIONS.\n"));
46: PETSCABORT(PETSC_COMM_SELF, PETSC_ERR_LIB);
47: }
48: }
49: devInitialized_ = true;
50: PetscFunctionReturn(PETSC_SUCCESS);
51: }
53: PetscErrorCode view(PetscViewer viewer) const noexcept
54: {
55: MPI_Comm comm;
56: PetscMPIInt rank;
57: PetscBool iascii;
59: PetscFunctionBegin;
60: PetscCheck(initialized(), PETSC_COMM_SELF, PETSC_ERR_COR, "Device %d being viewed before it was initialized or configured", id());
61: PetscCall(PetscObjectTypeCompare(reinterpret_cast<PetscObject>(viewer), PETSCVIEWERASCII, &iascii));
62: PetscCall(PetscObjectGetComm(reinterpret_cast<PetscObject>(viewer), &comm));
63: if (iascii) {
64: PetscViewer sviewer;
66: PetscCallMPI(MPI_Comm_rank(comm, &rank));
67: PetscCall(PetscViewerGetSubViewer(viewer, PETSC_COMM_SELF, &sviewer));
68: PetscCall(PetscViewerASCIIPrintf(sviewer, "[%d] device: %s\n", rank, syclDevice_.get_info<::sycl::info::device::name>().c_str()));
69: PetscCall(PetscViewerASCIIPushTab(sviewer));
70: PetscCall(PetscViewerASCIIPrintf(sviewer, "-> Device vendor: %s\n", syclDevice_.get_info<::sycl::info::device::vendor>().c_str()));
71: PetscCall(PetscViewerASCIIPopTab(sviewer));
72: PetscCall(PetscViewerFlush(sviewer));
73: PetscCall(PetscViewerRestoreSubViewer(viewer, PETSC_COMM_SELF, &sviewer));
74: PetscCall(PetscViewerFlush(viewer));
75: }
76: PetscFunctionReturn(PETSC_SUCCESS);
77: }
79: PetscErrorCode getattribute(PetscDeviceAttribute attr, void *value) const noexcept
80: {
81: PetscFunctionBegin;
82: PetscCheck(initialized(), PETSC_COMM_SELF, PETSC_ERR_COR, "Device %d not initialized", id());
83: switch (attr) {
84: case PETSC_DEVICE_ATTR_SIZE_T_SHARED_MEM_PER_BLOCK:
85: *static_cast<std::size_t *>(value) = syclDevice_.get_info<::sycl::info::device::local_mem_size>();
86: case PETSC_DEVICE_ATTR_MAX:
87: break;
88: }
89: PetscFunctionReturn(PETSC_SUCCESS);
90: }
92: private:
93: static ::sycl::device chooseSYCLDevice_(int id)
94: {
95: if (id == PETSC_SYCL_DEVICE_HOST) {
96: return ::sycl::device(::sycl::host_selector());
97: } else {
98: return ::sycl::device::get_devices(::sycl::info::device_type::gpu)[id];
99: }
100: }
102: // Is the underlying MPI aware of sycl (GPU) devices?
103: bool isMPISyclAware_() noexcept
104: {
105: const int bufSize = 2;
106: const int hbuf[bufSize] = {1, 0};
107: int *dbuf = nullptr;
108: bool awareness = false;
109: const auto SyclSignalHandler = [](int signal, void *ptr) -> PetscErrorCode {
110: if ((signal == SIGSEGV) && MPISyclAwareJumpBufferSet) std::longjmp(MPISyclAwareJumpBuffer, 1);
111: return PetscSignalHandlerDefault(signal, ptr);
112: };
114: PetscFunctionBegin;
115: auto Q = ::sycl::queue(syclDevice_);
116: dbuf = ::sycl::malloc_device<int>(bufSize, Q);
117: Q.memcpy(dbuf, hbuf, sizeof(int) * bufSize).wait();
118: PetscCallAbort(PETSC_COMM_SELF, PetscPushSignalHandler(SyclSignalHandler, nullptr));
119: MPISyclAwareJumpBufferSet = true;
120: if (setjmp(MPISyclAwareJumpBuffer)) {
121: // if a segv was triggered in the MPI_Allreduce below, it is very likely due to MPI not being GPU-aware
122: awareness = false;
123: PetscStackPop;
124: } else if (!MPI_Allreduce(dbuf, dbuf + 1, 1, MPI_INT, MPI_SUM, PETSC_COMM_SELF)) awareness = true;
125: MPISyclAwareJumpBufferSet = false;
126: PetscCallAbort(PETSC_COMM_SELF, PetscPopSignalHandler());
127: ::sycl::free(dbuf, Q);
128: PetscFunctionReturn(awareness);
129: }
130: };
132: PetscErrorCode Device::initialize(MPI_Comm comm, PetscInt *defaultDeviceId, PetscBool *defaultView, PetscDeviceInitType *defaultInitType) noexcept
133: {
134: auto id = *defaultDeviceId;
135: auto initType = *defaultInitType;
136: auto view = *defaultView, flg = PETSC_FALSE;
137: PetscInt ngpus;
139: PetscFunctionBegin;
140: if (initialized_) PetscFunctionReturn(PETSC_SUCCESS);
141: initialized_ = true;
142: PetscCall(PetscRegisterFinalize(finalize_));
143: PetscOptionsBegin(comm, nullptr, "PetscDevice sycl Options", "Sys");
144: PetscCall(base_type::PetscOptionDeviceInitialize(PetscOptionsObject, &initType, nullptr));
145: PetscCall(base_type::PetscOptionDeviceSelect(PetscOptionsObject, "Which sycl device to use? Pass -2 for host, PETSC_DECIDE (" PetscStringize(PETSC_DECIDE) ") to let PETSc decide, 0 and up for GPUs", "PetscDeviceCreate()", id, &id, nullptr, -2, std::numeric_limits<decltype(ngpus)>::max()));
146: static_assert(PETSC_DECIDE == -1, "Expect PETSC_DECIDE to be -1");
147: PetscCall(base_type::PetscOptionDeviceView(PetscOptionsObject, &view, &flg));
148: PetscOptionsEnd();
150: // post-process the options and lay the groundwork for initialization if needs be
151: std::vector<::sycl::device> gpu_devices = ::sycl::device::get_devices(::sycl::info::device_type::gpu);
152: ngpus = static_cast<PetscInt>(gpu_devices.size());
153: PetscCheck(ngpus || id < 0, comm, PETSC_ERR_USER_INPUT, "You specified a sycl gpu device with -device_select_sycl %d but there is no GPU", (int)id);
154: PetscCheck(ngpus <= 0 || id < ngpus, comm, PETSC_ERR_USER_INPUT, "You specified a sycl gpu device with -device_select_sycl %d but there are only %d GPU", (int)id, (int)ngpus);
156: if (initType == PETSC_DEVICE_INIT_NONE) id = PETSC_SYCL_DEVICE_NONE; /* user wants to disable all sycl devices */
157: else {
158: PetscCall(PetscDeviceCheckDeviceCount_Internal(ngpus));
159: if (id == PETSC_DECIDE) { /* petsc will choose a GPU device if any, otherwise a CPU device */
160: if (ngpus) {
161: PetscMPIInt rank;
162: PetscCallMPI(MPI_Comm_rank(comm, &rank));
163: id = rank % ngpus;
164: } else id = PETSC_SYCL_DEVICE_HOST;
165: }
166: if (view) initType = PETSC_DEVICE_INIT_EAGER;
167: }
169: if (id == -2) id = PETSC_SYCL_DEVICE_HOST; // user passed in '-device_select_sycl -2'. We transform it into canonical form
171: defaultDevice_ = static_cast<decltype(defaultDevice_)>(id);
172: PetscCheck(initType != PETSC_DEVICE_INIT_EAGER || id != PETSC_SYCL_DEVICE_NONE, comm, PETSC_ERR_USER_INPUT, "Cannot eagerly initialize sycl devices as you disabled them by -device_enable_sycl none");
173: // record the results of the initialization
174: *defaultDeviceId = id;
175: *defaultView = view;
176: *defaultInitType = initType;
177: PetscFunctionReturn(PETSC_SUCCESS);
178: }
180: PetscErrorCode Device::finalize_() noexcept
181: {
182: PetscFunctionBegin;
183: if (!initialized_) PetscFunctionReturn(PETSC_SUCCESS);
184: for (auto &&devPtr : devices_array_) delete devPtr;
185: defaultDevice_ = PETSC_SYCL_DEVICE_NONE; // disabled by default
186: initialized_ = false;
187: PetscFunctionReturn(PETSC_SUCCESS);
188: }
190: PetscErrorCode Device::init_device_id_(PetscInt *inid) const noexcept
191: {
192: const auto id = *inid == PETSC_DECIDE ? defaultDevice_ : *inid;
194: PetscFunctionBegin;
195: PetscCheck(defaultDevice_ != PETSC_SYCL_DEVICE_NONE, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Trying to retrieve a SYCL PetscDevice when it has been disabled");
196: PetscCheck(!(id < PETSC_SYCL_DEVICE_HOST) && !(id - PETSC_SYCL_DEVICE_HOST >= PETSC_DEVICE_MAX_DEVICES), PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Only supports %zu number of devices but trying to get device with id %" PetscInt_FMT,
197: devices_array_.size(), id);
198: if (!devices_[id]) devices_[id] = new DeviceInternal(id);
199: PetscCheck(id == devices_[id]->id(), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Entry %" PetscInt_FMT " contains device with mismatching id %" PetscInt_FMT, id, devices_[id]->id());
200: PetscCall(devices_[id]->initialize());
201: *inid = id;
202: PetscFunctionReturn(PETSC_SUCCESS);
203: }
205: PetscErrorCode Device::view_device_(PetscDevice device, PetscViewer viewer) noexcept
206: {
207: PetscFunctionBegin;
208: PetscCall(devices_[device->deviceId]->view(viewer));
209: PetscFunctionReturn(PETSC_SUCCESS);
210: }
212: PetscErrorCode Device::get_attribute_(PetscInt id, PetscDeviceAttribute attr, void *value) noexcept
213: {
214: PetscFunctionBegin;
215: PetscCall(devices_[id]->getattribute(attr, value));
216: PetscFunctionReturn(PETSC_SUCCESS);
217: }
219: } // namespace sycl
221: } // namespace device
223: } // namespace Petsc