Actual source code: sycldevice.sycl.cxx

  1: #include "sycldevice.hpp"
  2: #include <limits>  // for std::numeric_limits
  3: #include <csetjmp> // for MPI sycl device awareness
  4: #include <csignal> // SIGSEGV
  5: #include <vector>
  6: #include <CL/sycl.hpp>

  8: namespace Petsc
  9: {

 11: namespace device
 12: {

 14: namespace sycl
 15: {

 17: // definition for static
 18: std::array<Device::DeviceInternal *, PETSC_DEVICE_MAX_DEVICES> Device::devices_array_ = {};
 19: Device::DeviceInternal                                       **Device::devices_       = &Device::devices_array_[1];
 20: int                                                            Device::defaultDevice_ = PETSC_SYCL_DEVICE_NONE;
 21: bool                                                           Device::initialized_   = false;

 23: static std::jmp_buf MPISyclAwareJumpBuffer;
 24: static bool         MPISyclAwareJumpBufferSet;

 26: // internal "impls" class for SyclDevice. Each instance represents a single sycl device
 27: class PETSC_NODISCARD Device::DeviceInternal {
 28:   const int            id_; // -1 for the host device; 0 and up for gpu devices
 29:   bool                 devInitialized_;
 30:   const ::sycl::device syclDevice_;

 32: public:
 33:   // default constructor
 34:   DeviceInternal(int id) noexcept : id_(id), devInitialized_(false), syclDevice_(chooseSYCLDevice_(id)) { }
 35:   int  id() const { return id_; }
 36:   bool initialized() const { return devInitialized_; }

 38:   PetscErrorCode initialize() noexcept
 39:   {
 40:     PetscFunctionBegin;
 41:     if (initialized()) PetscFunctionReturn(PETSC_SUCCESS);
 42:     if (syclDevice_.is_gpu() && use_gpu_aware_mpi) {
 43:       if (!isMPISyclAware_()) {
 44:         PetscCall((*PetscErrorPrintf)("PETSc is configured with sycl support, but your MPI is not aware of sycl GPU devices. For better performance, please use a sycl GPU-aware MPI.\n"));
 45:         PetscCall((*PetscErrorPrintf)("If you do not care, add option -use_gpu_aware_mpi 0. To not see the message again, add the option to your .petscrc, OR add it to the env var PETSC_OPTIONS.\n"));
 46:         PETSCABORT(PETSC_COMM_SELF, PETSC_ERR_LIB);
 47:       }
 48:     }
 49:     devInitialized_ = true;
 50:     PetscFunctionReturn(PETSC_SUCCESS);
 51:   }

 53:   PetscErrorCode view(PetscViewer viewer) const noexcept
 54:   {
 55:     MPI_Comm    comm;
 56:     PetscMPIInt rank;
 57:     PetscBool   iascii;

 59:     PetscFunctionBegin;
 60:     PetscCheck(initialized(), PETSC_COMM_SELF, PETSC_ERR_COR, "Device %d being viewed before it was initialized or configured", id());
 61:     PetscCall(PetscObjectTypeCompare(reinterpret_cast<PetscObject>(viewer), PETSCVIEWERASCII, &iascii));
 62:     PetscCall(PetscObjectGetComm(reinterpret_cast<PetscObject>(viewer), &comm));
 63:     if (iascii) {
 64:       PetscViewer sviewer;

 66:       PetscCallMPI(MPI_Comm_rank(comm, &rank));
 67:       PetscCall(PetscViewerGetSubViewer(viewer, PETSC_COMM_SELF, &sviewer));
 68:       PetscCall(PetscViewerASCIIPrintf(sviewer, "[%d] device: %s\n", rank, syclDevice_.get_info<::sycl::info::device::name>().c_str()));
 69:       PetscCall(PetscViewerASCIIPushTab(sviewer));
 70:       PetscCall(PetscViewerASCIIPrintf(sviewer, "-> Device vendor: %s\n", syclDevice_.get_info<::sycl::info::device::vendor>().c_str()));
 71:       PetscCall(PetscViewerASCIIPopTab(sviewer));
 72:       PetscCall(PetscViewerFlush(sviewer));
 73:       PetscCall(PetscViewerRestoreSubViewer(viewer, PETSC_COMM_SELF, &sviewer));
 74:       PetscCall(PetscViewerFlush(viewer));
 75:     }
 76:     PetscFunctionReturn(PETSC_SUCCESS);
 77:   }

 79:   PetscErrorCode getattribute(PetscDeviceAttribute attr, void *value) const noexcept
 80:   {
 81:     PetscFunctionBegin;
 82:     PetscCheck(initialized(), PETSC_COMM_SELF, PETSC_ERR_COR, "Device %d not initialized", id());
 83:     switch (attr) {
 84:     case PETSC_DEVICE_ATTR_SIZE_T_SHARED_MEM_PER_BLOCK:
 85:       *static_cast<std::size_t *>(value) = syclDevice_.get_info<::sycl::info::device::local_mem_size>();
 86:     case PETSC_DEVICE_ATTR_MAX:
 87:       break;
 88:     }
 89:     PetscFunctionReturn(PETSC_SUCCESS);
 90:   }

 92: private:
 93:   static ::sycl::device chooseSYCLDevice_(int id)
 94:   {
 95:     if (id == PETSC_SYCL_DEVICE_HOST) {
 96:       return ::sycl::device(::sycl::host_selector());
 97:     } else {
 98:       return ::sycl::device::get_devices(::sycl::info::device_type::gpu)[id];
 99:     }
100:   }

102:   // Is the underlying MPI aware of sycl (GPU) devices?
103:   bool isMPISyclAware_() noexcept
104:   {
105:     const int  bufSize           = 2;
106:     const int  hbuf[bufSize]     = {1, 0};
107:     int       *dbuf              = nullptr;
108:     bool       awareness         = false;
109:     const auto SyclSignalHandler = [](int signal, void *ptr) -> PetscErrorCode {
110:       if ((signal == SIGSEGV) && MPISyclAwareJumpBufferSet) std::longjmp(MPISyclAwareJumpBuffer, 1);
111:       return PetscSignalHandlerDefault(signal, ptr);
112:     };

114:     PetscFunctionBegin;
115:     auto Q = ::sycl::queue(syclDevice_);
116:     dbuf   = ::sycl::malloc_device<int>(bufSize, Q);
117:     Q.memcpy(dbuf, hbuf, sizeof(int) * bufSize).wait();
118:     PetscCallAbort(PETSC_COMM_SELF, PetscPushSignalHandler(SyclSignalHandler, nullptr));
119:     MPISyclAwareJumpBufferSet = true;
120:     if (setjmp(MPISyclAwareJumpBuffer)) {
121:       // if a segv was triggered in the MPI_Allreduce below, it is very likely due to MPI not being GPU-aware
122:       awareness = false;
123:       PetscStackPop;
124:     } else if (!MPI_Allreduce(dbuf, dbuf + 1, 1, MPI_INT, MPI_SUM, PETSC_COMM_SELF)) awareness = true;
125:     MPISyclAwareJumpBufferSet = false;
126:     PetscCallAbort(PETSC_COMM_SELF, PetscPopSignalHandler());
127:     ::sycl::free(dbuf, Q);
128:     PetscFunctionReturn(awareness);
129:   }
130: };

132: PetscErrorCode Device::initialize(MPI_Comm comm, PetscInt *defaultDeviceId, PetscBool *defaultView, PetscDeviceInitType *defaultInitType) noexcept
133: {
134:   auto     id       = *defaultDeviceId;
135:   auto     initType = *defaultInitType;
136:   auto     view = *defaultView, flg = PETSC_FALSE;
137:   PetscInt ngpus;

139:   PetscFunctionBegin;
140:   if (initialized_) PetscFunctionReturn(PETSC_SUCCESS);
141:   initialized_ = true;
142:   PetscCall(PetscRegisterFinalize(finalize_));
143:   PetscOptionsBegin(comm, nullptr, "PetscDevice sycl Options", "Sys");
144:   PetscCall(base_type::PetscOptionDeviceInitialize(PetscOptionsObject, &initType, nullptr));
145:   PetscCall(base_type::PetscOptionDeviceSelect(PetscOptionsObject, "Which sycl device to use? Pass -2 for host, PETSC_DECIDE (" PetscStringize(PETSC_DECIDE) ") to let PETSc decide, 0 and up for GPUs", "PetscDeviceCreate()", id, &id, nullptr, -2, std::numeric_limits<decltype(ngpus)>::max()));
146:   static_assert(PETSC_DECIDE == -1, "Expect PETSC_DECIDE to be -1");
147:   PetscCall(base_type::PetscOptionDeviceView(PetscOptionsObject, &view, &flg));
148:   PetscOptionsEnd();

150:   // post-process the options and lay the groundwork for initialization if needs be
151:   std::vector<::sycl::device> gpu_devices = ::sycl::device::get_devices(::sycl::info::device_type::gpu);
152:   ngpus                                   = static_cast<PetscInt>(gpu_devices.size());
153:   PetscCheck(ngpus || id < 0, comm, PETSC_ERR_USER_INPUT, "You specified a sycl gpu device with -device_select_sycl %d but there is no GPU", (int)id);
154:   PetscCheck(ngpus <= 0 || id < ngpus, comm, PETSC_ERR_USER_INPUT, "You specified a sycl gpu device with -device_select_sycl %d but there are only %d GPU", (int)id, (int)ngpus);

156:   if (initType == PETSC_DEVICE_INIT_NONE) id = PETSC_SYCL_DEVICE_NONE; /* user wants to disable all sycl devices */
157:   else {
158:     PetscCall(PetscDeviceCheckDeviceCount_Internal(ngpus));
159:     if (id == PETSC_DECIDE) { /* petsc will choose a GPU device if any, otherwise a CPU device */
160:       if (ngpus) {
161:         PetscMPIInt rank;
162:         PetscCallMPI(MPI_Comm_rank(comm, &rank));
163:         id = rank % ngpus;
164:       } else id = PETSC_SYCL_DEVICE_HOST;
165:     }
166:     if (view) initType = PETSC_DEVICE_INIT_EAGER;
167:   }

169:   if (id == -2) id = PETSC_SYCL_DEVICE_HOST; // user passed in '-device_select_sycl -2'. We transform it into canonical form

171:   defaultDevice_ = static_cast<decltype(defaultDevice_)>(id);
172:   PetscCheck(initType != PETSC_DEVICE_INIT_EAGER || id != PETSC_SYCL_DEVICE_NONE, comm, PETSC_ERR_USER_INPUT, "Cannot eagerly initialize sycl devices as you disabled them by -device_enable_sycl none");
173:   // record the results of the initialization
174:   *defaultDeviceId = id;
175:   *defaultView     = view;
176:   *defaultInitType = initType;
177:   PetscFunctionReturn(PETSC_SUCCESS);
178: }

180: PetscErrorCode Device::finalize_() noexcept
181: {
182:   PetscFunctionBegin;
183:   if (!initialized_) PetscFunctionReturn(PETSC_SUCCESS);
184:   for (auto &&devPtr : devices_array_) delete devPtr;
185:   defaultDevice_ = PETSC_SYCL_DEVICE_NONE; // disabled by default
186:   initialized_   = false;
187:   PetscFunctionReturn(PETSC_SUCCESS);
188: }

190: PetscErrorCode Device::init_device_id_(PetscInt *inid) const noexcept
191: {
192:   const auto id = *inid == PETSC_DECIDE ? defaultDevice_ : *inid;

194:   PetscFunctionBegin;
195:   PetscCheck(defaultDevice_ != PETSC_SYCL_DEVICE_NONE, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Trying to retrieve a SYCL PetscDevice when it has been disabled");
196:   PetscCheck(!(id < PETSC_SYCL_DEVICE_HOST) && !(id - PETSC_SYCL_DEVICE_HOST >= PETSC_DEVICE_MAX_DEVICES), PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Only supports %zu number of devices but trying to get device with id %" PetscInt_FMT,
197:              devices_array_.size(), id);
198:   if (!devices_[id]) devices_[id] = new DeviceInternal(id);
199:   PetscCheck(id == devices_[id]->id(), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Entry %" PetscInt_FMT " contains device with mismatching id %" PetscInt_FMT, id, devices_[id]->id());
200:   PetscCall(devices_[id]->initialize());
201:   *inid = id;
202:   PetscFunctionReturn(PETSC_SUCCESS);
203: }

205: PetscErrorCode Device::view_device_(PetscDevice device, PetscViewer viewer) noexcept
206: {
207:   PetscFunctionBegin;
208:   PetscCall(devices_[device->deviceId]->view(viewer));
209:   PetscFunctionReturn(PETSC_SUCCESS);
210: }

212: PetscErrorCode Device::get_attribute_(PetscInt id, PetscDeviceAttribute attr, void *value) noexcept
213: {
214:   PetscFunctionBegin;
215:   PetscCall(devices_[id]->getattribute(attr, value));
216:   PetscFunctionReturn(PETSC_SUCCESS);
217: }

219: } // namespace sycl

221: } // namespace device

223: } // namespace Petsc