bes Updated for version 3.20.13
HDF5Array.cc
Go to the documentation of this file.
1// This file is part of hdf5_handler a HDF5 file handler for the OPeNDAP
2// data server.
3
4// Author: Hyo-Kyung Lee <hyoklee@hdfgroup.org> and Muqun Yang
5// <myang6@hdfgroup.org>
6
7// Copyright (c) 2009-2016 The HDF Group, Inc. and OPeNDAP, Inc.
8//
9// This is H5free_memory software; you can redistribute it and/or modify it under the
10// terms of the GNU Lesser General Public License as published by the Free
11// Software Foundation; either version 2.1 of the License, or (at your
12// option) any later version.
13//
14// This software is distributed in the hope that it will be useful, but
15// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
16// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
17// License for more details.
18//
19// You should have received a copy of the GNU Lesser General Public
20// License along with this library; if not, write to the Free Software
21// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22//
23// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
24// You can contact The HDF Group, Inc. at 1800 South Oak Street,
25// Suite 203, Champaign, IL 61820
33
34
35#include "config_hdf5.h"
36
37#include <iostream>
38#include <memory>
39#include <sstream>
40#include <algorithm>
41#include <ctype.h>
42
43#include <BESDebug.h>
44#include <libdap/Error.h>
45#include <libdap/InternalErr.h>
46
47#include "HDF5Array.h"
48#include "HDF5Structure.h"
49#include "HDF5Str.h"
50
51using namespace std;
52using namespace libdap;
53
55 return new HDF5Array(*this);
56}
57
58HDF5Array::HDF5Array(const string & n, const string &d, BaseType * v) :
59 Array(n, d, v) {
60}
61
62
63int HDF5Array::format_constraint(int *offset, int *step, int *count) {
64
65 // For the 0-length array case, just return 0.
66 if(length() == 0)
67 return 0;
68
69 int nels = 1;
70 int id = 0;
71
72 Dim_iter p = dim_begin();
73
74 while (p != dim_end()) {
75
76 int start = dimension_start(p, true);
77 int stride = dimension_stride(p, true);
78 int stop = dimension_stop(p, true);
79
80 // Check for empty constraint
81 if (start > stop) {
82 ostringstream oss;
83
84 oss << "Array/Grid hyperslab start point "<< start <<
85 " is greater than stop point " << stop <<".";
86 throw Error(malformed_expr, oss.str());
87 }
88
89 offset[id] = start;
90 step[id] = stride;
91 count[id] = ((stop - start) / stride) + 1; // count of elements
92 nels *= count[id]; // total number of values for variable
93
94 BESDEBUG("h5",
95 "=format_constraint():"
96 << "id=" << id << " offset=" << offset[id]
97 << " step=" << step[id]
98 << " count=" << count[id]
99 << endl);
100
101 id++;
102 p++;
103 }
104
105 return nels;
106}
107
108
110{
111 BESDEBUG("h5",
112 ">read() dataset=" << dataset()
113 << " dimension=" << d_num_dim
114 << " data_size=" << d_memneed << " length=" << length()
115 << endl);
116
117 hid_t file_id = H5Fopen(dataset().c_str(),H5F_ACC_RDONLY,H5P_DEFAULT);
118
119 BESDEBUG("h5","after H5Fopen "<<endl);
120 BESDEBUG("h5","variable name is "<<name() <<endl);
121 BESDEBUG("h5","variable path is "<<var_path <<endl);
122
123 hid_t dset_id = -1;
124
125 if(true == is_dap4())
126 dset_id = H5Dopen2(file_id,var_path.c_str(),H5P_DEFAULT);
127 else
128 dset_id = H5Dopen2(file_id,name().c_str(),H5P_DEFAULT);
129
130 BESDEBUG("h5","after H5Dopen2 "<<endl);
131 // Leave the following code. We may replace the struct DS and DSattr(see hdf5_handler.h)
132#if 0
133 hid_t dspace_id = H5Dget_space(dset_id);
134 if(dspace_id < 0) {
135 H5Dclose(dset_id);
136 H5Fclose(file_id);
137 throw InternalErr(__FILE__,__LINE__, "Fail to obtain the dataspace .");
138 }
139
140 int num_dim = H5Sget_simple_extent_ndims(dspace_id);
141 if(num_dim < 0) {
142 H5Sclose(dspace_id);
143 H5Dclose(dset_id);
144 H5Fclose(file_id);
145 throw InternalErr(__FILE__,__LINE__, "Fail to obtain the datatype .");
146 }
147
148 H5Sclose(dspace_id);
149#endif
150
151 hid_t dtype_id = H5Dget_type(dset_id);
152 if(dtype_id < 0) {
153 H5Dclose(dset_id);
154 H5Fclose(file_id);
155 throw InternalErr(__FILE__,__LINE__, "Fail to obtain the datatype .");
156 }
157
158
159 vector<int> offset(d_num_dim);
160 vector<int> count(d_num_dim);
161 vector<int> step(d_num_dim);
162 int nelms = format_constraint(offset.data(), step.data(), count.data()); // Throws Error.
163 vector<char>values;
164
165
166 // We only map the reference to URL when the dataset is an array of reference.
167 if (get_dap_type(dtype_id,is_dap4()) == "Url") {
168 bool ret_ref = false;
169 try {
170 ret_ref = m_array_of_reference(dset_id,dtype_id);
171 H5Tclose(dtype_id);
172 H5Dclose(dset_id);
173 H5Fclose(file_id);
174
175 }
176 catch(...) {
177 H5Tclose(dtype_id);
178 H5Dclose(dset_id);
179 H5Fclose(file_id);
180 throw;
181
182 }
183 return ret_ref;
184 }
185
186 try {
187 do_array_read(dset_id,dtype_id,values,false,0,nelms,offset.data(),count.data(),step.data());
188 }
189 catch(...) {
190 H5Tclose(dtype_id);
191 H5Dclose(dset_id);
192 H5Fclose(file_id);
193 throw;
194 }
195
196 H5Tclose(dtype_id);
197 H5Dclose(dset_id);
198 H5Fclose(file_id);
199
200 return true;
201}
202
203void HDF5Array::do_array_read(hid_t dset_id,hid_t dtype_id,vector<char>&values,bool has_values,int values_offset,
204 int nelms,int* offset,int* count, int* step)
205{
206
207 H5T_class_t tcls = H5Tget_class(dtype_id);
208
209 if(H5T_COMPOUND == tcls)
210 m_array_of_structure(dset_id,values,has_values,values_offset,nelms,offset,count,step);
211 else if(H5T_INTEGER == tcls || H5T_FLOAT == tcls || H5T_STRING == tcls)
212 m_array_of_atomic(dset_id,dtype_id,nelms,offset,count,step);
213 else {
214 throw InternalErr(__FILE__,__LINE__,"Fail to read the data for Unsupported datatype.");
215 }
216
217}
218
219void HDF5Array:: m_array_of_atomic(hid_t dset_id, hid_t dtype_id,
220 int nelms,int* offset,int* count, int* step)
221{
222
223 hid_t memtype = -1;
224 if((memtype = H5Tget_native_type(dtype_id, H5T_DIR_ASCEND))<0) {
225 throw InternalErr (__FILE__, __LINE__, "Fail to obtain memory datatype.");
226 }
227
228 // First handle variable-length string
229 if (H5Tis_variable_str(memtype) && H5Tget_class(memtype) == H5T_STRING) {
230
231 vector<hsize_t> hoffset;
232 vector<hsize_t>hcount;
233 vector<hsize_t>hstep;
234 hoffset.resize(d_num_dim);
235 hcount.resize(d_num_dim);
236 hstep.resize(d_num_dim);
237 for (int i = 0; i <d_num_dim; i++) {
238 hoffset[i] = (hsize_t) offset[i];
239 hcount[i] = (hsize_t) count[i];
240 hstep[i] = (hsize_t) step[i];
241 }
242
243 vector<string>finstrval;
244 finstrval.resize(nelms);
245 try {
246 read_vlen_string(dset_id, nelms, hoffset.data(), hstep.data(), hcount.data(),finstrval);
247 }
248 catch(...) {
249 H5Tclose(memtype);
250 throw InternalErr(__FILE__,__LINE__,"Fail to read variable-length string.");
251 }
252 set_value(finstrval,nelms);
253 H5Tclose(memtype);
254 return ;
255 }
256
257 try {
258 if (nelms == d_num_elm) {
259
260 vector<char> convbuf(d_memneed);
261 get_data(dset_id, (void *) convbuf.data());
262
263 // Check if a Signed Byte to Int16 conversion is necessary, this is only valid for DAP2.
264 if(false == is_dap4()) {
265 if (1 == H5Tget_size(memtype) && H5T_SGN_2 == H5Tget_sign(memtype))
266 {
267 vector<short> convbuf2(nelms);
268 for (int i = 0; i < nelms; i++) {
269 convbuf2[i] = (signed char) (convbuf[i]);
270 BESDEBUG("h5", "convbuf[" << i << "]="
271 << (signed char)convbuf[i] << endl);
272 BESDEBUG("h5", "convbuf2[" << i << "]="
273 << convbuf2[i] << endl)
274 ;
275 }
276 // Libdap will generate the wrong output.
277 m_intern_plain_array_data((char*) convbuf2.data(),memtype);
278 }
279 else
280 m_intern_plain_array_data(convbuf.data(),memtype);
281 }
282 else
283 m_intern_plain_array_data(convbuf.data(),memtype);
284 } // end of "if (nelms == d_num_elm)"
285 else {
286 size_t data_size = nelms * H5Tget_size(memtype);
287 if (data_size == 0) {
288 throw InternalErr(__FILE__, __LINE__, "get_size failed");
289 }
290 vector<char> convbuf(data_size);
291 get_slabdata(dset_id, offset, step, count, d_num_dim, convbuf.data());
292
293 // Check if a Signed Byte to Int16 conversion is necessary.
294 if(false == is_dap4()){
295 if (1 == H5Tget_size(memtype) && H5T_SGN_2 == H5Tget_sign(memtype)) {
296 vector<short> convbuf2(data_size);
297 for (int i = 0; i < (int)data_size; i++) {
298 convbuf2[i] = static_cast<signed char> (convbuf[i]);
299 }
300 m_intern_plain_array_data((char*) convbuf2.data(),memtype);
301 }
302 else {
303 m_intern_plain_array_data(convbuf.data(),memtype);
304 }
305 }
306 else
307 m_intern_plain_array_data(convbuf.data(),memtype);
308
309 }
310 H5Tclose(memtype);
311 }
312 catch (...) {
313 H5Tclose(memtype);
314 throw;
315 }
316
317}
318
319bool HDF5Array::m_array_of_structure(hid_t dsetid, vector<char>&values,bool has_values,int values_offset,
320 int nelms,const int* offset,const int* count, const int* step) {
321
322 BESDEBUG("h5", "=read() Array of Structure length=" << length() << endl);
323
324 hid_t mspace = -1;
325 hid_t memtype = -1;
326 hid_t dtypeid = -1;
327 size_t ty_size = -1;
328
329 if((dtypeid = H5Dget_type(dsetid)) < 0)
330 throw InternalErr (__FILE__, __LINE__, "Cannot obtain the datatype.");
331
332 if((memtype = H5Tget_native_type(dtypeid, H5T_DIR_ASCEND))<0) {
333 H5Tclose(dtypeid);
334 throw InternalErr (__FILE__, __LINE__, "Fail to obtain memory datatype.");
335 }
336
337 ty_size = H5Tget_size(memtype);
338 if (ty_size == 0) {
339 H5Tclose(memtype);
340 H5Tclose(dtypeid);
341 throw InternalErr (__FILE__, __LINE__,"Fail to obtain the size of HDF5 compound datatype.");
342 }
343
344 if(false == has_values) {
345
346 hid_t dspace = -1;
347
348 if ((dspace = H5Dget_space(dsetid))<0) {
349 H5Tclose(memtype);
350 H5Tclose(dtypeid);
351 throw InternalErr (__FILE__, __LINE__, "Cannot obtain data space.");
352 }
353
354 d_num_dim = H5Sget_simple_extent_ndims(dspace);
355 if(d_num_dim < 0) {
356 H5Tclose(memtype);
357 H5Tclose(dtypeid);
358 H5Sclose(dspace);
359 throw InternalErr (__FILE__, __LINE__, "Cannot obtain the number of dimensions of the data space.");
360 }
361
362 vector<hsize_t> hoffset;
363 vector<hsize_t>hcount;
364 vector<hsize_t>hstep;
365 hoffset.resize(d_num_dim);
366 hcount.resize(d_num_dim);
367 hstep.resize(d_num_dim);
368 for (int i = 0; i <d_num_dim; i++) {
369 hoffset[i] = (hsize_t) offset[i];
370 hcount[i] = (hsize_t) count[i];
371 hstep[i] = (hsize_t) step[i];
372 }
373
374 if (H5Sselect_hyperslab(dspace, H5S_SELECT_SET,
375 hoffset.data(), hstep.data(),
376 hcount.data(), nullptr) < 0) {
377 H5Tclose(memtype);
378 H5Tclose(dtypeid);
379 H5Sclose(dspace);
380 throw InternalErr (__FILE__, __LINE__, "Cannot generate the hyperslab of the HDF5 dataset.");
381 }
382
383 mspace = H5Screate_simple(d_num_dim, hcount.data(),nullptr);
384 if (mspace < 0) {
385 H5Sclose(dspace);
386 H5Tclose(memtype);
387 H5Tclose(dtypeid);
388 throw InternalErr (__FILE__, __LINE__, "Cannot create the memory space.");
389 }
390
391 values.resize(nelms*ty_size);
392 hid_t read_ret = -1;
393 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,(void*)values.data());
394 if (read_ret < 0) {
395 H5Tclose(memtype);
396 H5Tclose(dtypeid);
397 H5Sclose(dspace);
398 throw InternalErr (__FILE__, __LINE__, "Fail to read the HDF5 compound datatype dataset.");
399 }
400
401 H5Sclose(dspace);
402 has_values = true;
403 } // end of "if(false == has_values)" block
404
405 HDF5Structure *h5s = nullptr;
406 hid_t memb_id = -1;
407 char* memb_name = nullptr;
408
409 try {
410
411 // Loop through all the elements in this compound datatype array.
412 for (int element = 0; element < nelms; ++element) {
413
414 h5s = dynamic_cast<HDF5Structure*>(var()->ptr_duplicate());
415 H5T_class_t memb_cls = H5T_NO_CLASS;
416 int nmembs = 0;
417 size_t memb_offset = 0;
418
419 if((nmembs = H5Tget_nmembers(memtype)) < 0)
420 throw InternalErr (__FILE__, __LINE__, "Fail to obtain number of HDF5 compound datatype.");
421
422 for(unsigned int u = 0; u < (unsigned)nmembs; u++) {
423
424 // Get member type ID
425 if((memb_id = H5Tget_member_type(memtype, u)) < 0)
426 throw InternalErr (__FILE__, __LINE__, "Fail to obtain the datatype of an HDF5 compound datatype member.");
427
428 // Get member type class
429 if((memb_cls = H5Tget_member_class (memtype, u)) < 0)
430 throw InternalErr (__FILE__, __LINE__, "Fail to obtain the datatype class of an HDF5 compound datatype member.");
431
432 // Get member offset,H5Tget_member_offset only fails
433 // when H5Tget_memeber_class fails. Sinc H5Tget_member_class
434 // is checked above. So no need to check the return value.
435 memb_offset= H5Tget_member_offset(memtype,u);
436
437 // Get member name
438 memb_name = H5Tget_member_name(memtype,u);
439 if(memb_name == nullptr)
440 throw InternalErr (__FILE__, __LINE__, "Fail to obtain the name of an HDF5 compound datatype member.");
441
442 BaseType *field = h5s->var(memb_name);
443 if (memb_cls == H5T_COMPOUND) {
444 HDF5Structure &memb_h5s = dynamic_cast<HDF5Structure&>(*field);
445 memb_h5s.do_structure_read(dsetid, memb_id,values,has_values,memb_offset+values_offset+ty_size*element);
446 }
447
448 else if(memb_cls == H5T_ARRAY) {
449
450 // memb_id, obtain the number of dimensions
451 int at_ndims = H5Tget_array_ndims(memb_id);
452 if(at_ndims <= 0)
453 throw InternalErr (__FILE__, __LINE__, "Fail to obtain number of dimensions of the array datatype.");
454
455#if 0
456 //HDF5Array &h5_array_type = dynamic_cast<HDF5Array&>(*h5s->var(memb_name));
457#endif
458 HDF5Array &h5_array_type = dynamic_cast<HDF5Array&>(*field);
459 vector<int> at_offset(at_ndims,0);
460 vector<int> at_count(at_ndims,0);
461 vector<int> at_step(at_ndims,0);
462
463 int at_nelms = h5_array_type.format_constraint(at_offset.data(),at_step.data(),at_count.data());
464
465 // Read the array data
466 h5_array_type.do_h5_array_type_read(dsetid,memb_id,values,has_values,memb_offset+values_offset+ty_size*element,
467 at_nelms,at_offset.data(),at_count.data(),at_step.data());
468
469 }
470 else if(memb_cls == H5T_INTEGER || memb_cls == H5T_FLOAT) {
471
472 if(true == promote_char_to_short(memb_cls,memb_id)) {
473 void *src = (void*)(values.data() + (element*ty_size) + values_offset +memb_offset);
474 char val_int8;
475 memcpy(&val_int8,src,1);
476 short val_short=(short)val_int8;
477 field->val2buf(&val_short);
478 }
479 else {
480 field->val2buf(values.data() + (element*ty_size) + values_offset +memb_offset);
481 }
482
483 }
484 else if(memb_cls == H5T_STRING) {
485
486 // distinguish between variable length and fixed length
487 if(true == H5Tis_variable_str(memb_id)) {
488 void *src = (void*)(values.data()+(element*ty_size)+values_offset + memb_offset);
489 string final_str;
490 get_vlen_str_data((char*)src,final_str);
491 field->val2buf(&final_str);
492 }
493 else {// Obtain fixed-size string value
494 void *src = (void*)(values.data()+(element*ty_size)+values_offset + memb_offset);
495 vector<char> str_val;
496 size_t memb_size = H5Tget_size(memb_id);
497 if (memb_size == 0) {
498 H5Tclose(memb_id);
499 H5free_memory(memb_name);
500 delete h5s;
501 throw InternalErr (__FILE__, __LINE__,"Fail to obtain the size of HDF5 compound datatype.");
502 }
503 str_val.resize(memb_size);
504 memcpy(str_val.data(),src,memb_size);
505 string temp_string(str_val.begin(),str_val.end());
506 field->val2buf(&temp_string);
507 }
508 }
509 else {
510 H5free_memory(memb_name);
511 H5Tclose(memb_id);
512 delete h5s;
513 throw InternalErr (__FILE__, __LINE__,
514 "Only support the field of compound datatype when the field type class is integer, float, string, array or compound..");
515
516 }
517
518 // Close member type ID
519 H5Tclose(memb_id);
520 H5free_memory(memb_name);
521 field->set_read_p(true);
522 } // end "for(unsigned u = 0)"
523 h5s->set_read_p(true);
524 set_vec(element,h5s);
525 delete h5s;
526 } // end "for (int element=0"
527
528 if(true == has_values) {
529 if(-1 == mspace)
530 throw InternalErr(__FILE__, __LINE__, "memory type and memory space for this compound datatype should be valid.");
531
532 if(H5Dvlen_reclaim(memtype,mspace,H5P_DEFAULT,(void*)values.data())<0)
533 throw InternalErr(__FILE__, __LINE__, "Unable to reclaim the compound datatype array.");
534 H5Sclose(mspace);
535
536 }
537
538 H5Tclose(dtypeid);
539 H5Tclose(memtype);
540
541 }
542 catch(...) {
543
544 if(memb_id != -1)
545 H5Tclose(memb_id);
546 if(memb_name != nullptr)
547 H5free_memory(memb_name);
548 if(h5s != nullptr)
549 delete h5s;
550 if(true == has_values) {
551 if(H5Dvlen_reclaim(memtype,mspace,H5P_DEFAULT,(void*)(values.data()))<0) {
552 H5Tclose(memtype);
553 H5Sclose(mspace);
554 }
555 H5Sclose(mspace);
556 }
557 H5Tclose(memtype);
558 H5Tclose(dtypeid);
559 throw;
560 }
561
562 set_read_p(true);
563
564 return false;
565}
566
567// Haven't checked the codes and comments
568// Haven't added the close handles routines for error handlings yet. KY 2011-11-18
569bool HDF5Array::m_array_of_reference(hid_t dset_id,hid_t dtype_id)
570{
571
572#if (H5_VERS_MAJOR == 1 && (H5_VERS_MINOR == 10 || H5_VERS_MINOR == 8 || H5_VERS_MINOR == 6))
573 hid_t d_dset_id = dset_id;
574 hdset_reg_ref_t *rbuf = nullptr;
575
576 try {
577
578 vector<int> offset(d_num_dim);
579 vector<int> count(d_num_dim);
580 vector<int> step(d_num_dim);
581
582
583 int nelms = format_constraint(offset.data(), step.data(), count.data()); // Throws Error.
584 vector<string> v_str(nelms);
585
586 BESDEBUG("h5", "=read() URL type is detected. "
587 << "nelms=" << nelms << " full_size=" << d_num_elm << endl);
588
589
590 // Handle regional reference.
591 if (H5Tequal(dtype_id, H5T_STD_REF_DSETREG) < 0) {
592 throw InternalErr(__FILE__, __LINE__, "H5Tequal() failed");
593 }
594
595 if (H5Tequal(dtype_id, H5T_STD_REF_DSETREG) > 0) {
596 BESDEBUG("h5", "=read() Got regional reference. " << endl);
597 // Vector doesn't work for this case. somehow it doesn't support the type.
598 rbuf = new hdset_reg_ref_t[d_num_elm];
599 if(rbuf == nullptr){
600 throw InternalErr(__FILE__, __LINE__, "new() failed.");
601 }
602 if (H5Dread(d_dset_id, H5T_STD_REF_DSETREG, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) {
603 throw InternalErr(__FILE__, __LINE__, "H5Dread() failed.");
604 }
605
606 for (int i = 0; i < nelms; i++) {
607 // Let's assume that URL array is always 1 dimension.
608 BESDEBUG("h5", "=read() rbuf[" << i << "]" <<
609 rbuf[offset[0] + i * step[0]] << endl);
610
611 if (rbuf[offset[0] + i * step[0]][0] != '\0') {
612 char r_name[DODS_NAMELEN];
613
614 hid_t did_r = H5RDEREFERENCE(d_dset_id, H5R_DATASET_REGION, rbuf[offset[0] + i * step[0]]);
615 if (did_r < 0) {
616 throw InternalErr(__FILE__, __LINE__, "H5RDEREFERENCE() failed.");
617
618 }
619
620 if (H5Iget_name(did_r, (char *) r_name, DODS_NAMELEN) < 0) {
621 throw InternalErr(__FILE__, __LINE__, "H5Iget_name() failed.");
622 }
623 BESDEBUG("h5", "=read() dereferenced name is " << r_name
624 << endl);
625
626 string varname(r_name);
627 hid_t space_id = H5Rget_region(did_r, H5R_DATASET_REGION, rbuf[offset[0] + i * step[0]]);
628 if (space_id < 0) {
629 throw InternalErr(__FILE__, __LINE__, "H5Rget_region() failed.");
630
631 }
632
633 int ndim = H5Sget_simple_extent_ndims(space_id);
634 if (ndim < 0) {
635 throw InternalErr(__FILE__, __LINE__, "H5Sget_simple_extent_ndims() failed.");
636 }
637
638 BESDEBUG("h5", "=read() dim is " << ndim << endl);
639
640 string expression;
641 switch (H5Sget_select_type(space_id)) {
642
643 case H5S_SEL_NONE:
644 BESDEBUG("h5", "=read() None selected." << endl);
645 break;
646
647 case H5S_SEL_POINTS: {
648 BESDEBUG("h5", "=read() Points selected." << endl);
649 hssize_t npoints = H5Sget_select_npoints(space_id);
650 if (npoints < 0) {
651 throw InternalErr(__FILE__, __LINE__,
652 "Cannot determine number of elements in the dataspace selection");
653 }
654
655 BESDEBUG("h5", "=read() npoints are " << npoints
656 << endl);
657 vector<hsize_t> buf(npoints * ndim);
658 if (H5Sget_select_elem_pointlist(space_id, 0, npoints, buf.data()) < 0) {
659 throw InternalErr(__FILE__, __LINE__, "H5Sget_select_elem_pointlist() failed.");
660 }
661
662#if 0
663 for (int j = 0; j < npoints * ndim; j++) {
664 "h5", "=read() npoints buf[0] =" << buf[j] <<endl;
665 }
666#endif
667
668 for (int j = 0; j < (int) npoints; j++) {
669 // Name of the dataset.
670 expression.append(varname);
671 for (int k = 0; k < ndim; k++) {
672 ostringstream oss;
673 oss << "[" << (int) buf[j * ndim + k] << "]";
674 expression.append(oss.str());
675 }
676 if (j != (int) (npoints - 1)) {
677 expression.append(",");
678 }
679 }
680 v_str[i].append(expression);
681
682 break;
683 }
684 case H5S_SEL_HYPERSLABS: {
685 vector<hsize_t> start(ndim);
686 vector<hsize_t> end(ndim);
687 vector<hsize_t>stride(ndim);
688 vector<hsize_t>s_count(ndim);
689 vector<hsize_t>block(ndim);
690
691 BESDEBUG("h5", "=read() Slabs selected." << endl);
692 BESDEBUG("h5", "=read() nblock is " <<
693 H5Sget_select_hyper_nblocks(space_id) << endl);
694
695#if (H5_VERS_MAJOR == 1 && H5_VERS_MINOR == 8)
696 if (H5Sget_select_bounds(space_id, start.data(), end.data()) < 0) {
697 throw InternalErr(__FILE__, __LINE__, "H5Sget_select_bounds() failed.");
698 }
699#else
700 if (H5Sget_regular_hyperslab(space_id, start.data(), stride.data(), s_count.data(), block.data()) < 0) {
701 throw InternalErr(__FILE__, __LINE__, "H5Sget_regular_hyperslab() failed.");
702 }
703#endif
704
705 for (int j = 0; j < ndim; j++) {
706 ostringstream oss;
707 BESDEBUG("h5", "start " << start[j]
708 << "stride "<<stride[j]
709 << "count "<< s_count[j]
710 << "block "<< block[j]
711 <<endl);
712
713 // Map from HDF5's start,stride,count,block to DAP's start,stride,end.
714 end[j] = start[j] + stride[j]*(s_count[j]-1)+(block[j]-1);
715 BESDEBUG("h5", "=read() start is " << start[j]
716 << "=read() end is " << end[j] << endl);
717 oss << "[" << start[j] << ":" << stride[j] << ":" << end[j] << "]";
718 expression.append(oss.str());
719 BESDEBUG("h5", "=read() expression is "
720 << expression << endl)
721 ;
722 }
723 v_str[i] = varname;
724 if (!expression.empty()) {
725 v_str[i].append(expression);
726 }
727 // Constraint expression. [start:1:end]
728 break;
729 }
730 case H5S_SEL_ALL:
731 BESDEBUG("h5", "=read() All selected." << endl);
732 break;
733
734 default:
735 BESDEBUG("h5", "Unknown space type." << endl);
736 break;
737 }
738
739 }
740 else {
741 v_str[i] = "";
742 }
743 }
744 delete[] rbuf;
745 }
746
747 // Handle object reference.
748 if (H5Tequal(dtype_id, H5T_STD_REF_OBJ) < 0) {
749 throw InternalErr(__FILE__, __LINE__, "H5Tequal() failed.");
750 }
751
752 if (H5Tequal(dtype_id, H5T_STD_REF_OBJ) > 0) {
753 BESDEBUG("h5", "=read() Got object reference. " << endl);
754 vector<hobj_ref_t> orbuf;
755 orbuf.resize(d_num_elm);
756 if (H5Dread(d_dset_id, H5T_STD_REF_OBJ, H5S_ALL, H5S_ALL, H5P_DEFAULT, orbuf.data()) < 0) {
757 throw InternalErr(__FILE__, __LINE__, "H5Dread failed()");
758 }
759
760 for (int i = 0; i < nelms; i++) {
761 // Let's assume that URL array is always 1 dimension.
762 hid_t did_r = H5RDEREFERENCE(d_dset_id, H5R_OBJECT, &orbuf[offset[0] + i * step[0]]);
763 if (did_r < 0) {
764 throw InternalErr(__FILE__, __LINE__, "H5RDEREFERENCE() failed.");
765 }
766 char r_name[DODS_NAMELEN];
767 if (H5Iget_name(did_r, (char *) r_name, DODS_NAMELEN) < 0) {
768 throw InternalErr(__FILE__, __LINE__, "H5Iget_name() failed.");
769 }
770
771 // Shorten the dataset name
772 string varname(r_name);
773
774 BESDEBUG("h5", "=read() dereferenced name is " << r_name <<endl);
775 v_str[i] = varname;
776 }
777 }
778 set_value(v_str.data(), nelms);
779 return false;
780 }
781 catch (...) {
782 if(rbuf!= nullptr)
783 delete[] rbuf;
784 throw;
785 }
786#else
787 return m_array_of_reference_new_h5_apis(dset_id,dtype_id);
788
789#endif
790}
791
792bool HDF5Array::m_array_of_reference_new_h5_apis(hid_t dset_id,hid_t dtype_id) {
793
794#if (H5_VERS_MAJOR == 1 && (H5_VERS_MINOR == 10 || H5_VERS_MINOR == 8 || H5_VERS_MINOR == 6))
795 throw InternalErr(__FILE__, __LINE__,
796 "The HDF5 handler compiled with earlier version (<=110)of the HDF5 library should not call method that uses new reference APIs");
797 return false;
798#else
799
800 H5R_ref_t *rbuf = nullptr;
801 hid_t mem_space_id;
802 hid_t file_space_id;
803
804 try {
805
806 // First we need to read the reference data from DAP's hyperslab selection.
807 vector<int> offset(d_num_dim);
808 vector<int> count(d_num_dim);
809 vector<int> step(d_num_dim);
810 vector<hsize_t> hoffset(d_num_dim);
811 vector<hsize_t>hcount(d_num_dim);
812 vector<hsize_t>hstep(d_num_dim);
813
814 int nelms = format_constraint(offset.data(), step.data(), count.data());
815 for (int i = 0; i <d_num_dim; i++) {
816 hoffset[i] = (hsize_t) offset[i];
817 hcount[i] = (hsize_t) count[i];
818 hstep[i] = (hsize_t) step[i];
819 }
820
821 BESDEBUG("h5", "=read() URL type is detected. "
822 << "nelms=" << nelms << endl);
823
824 rbuf = new H5R_ref_t[nelms];
825
826 file_space_id = H5Dget_space(dset_id);
827 if(file_space_id < 0)
828 throw InternalErr(__FILE__, __LINE__, "Fail to obtain reference dataset file space.");
829
830 if (H5Sselect_hyperslab(file_space_id, H5S_SELECT_SET,
831 hoffset.data(), hstep.data(),
832 hcount.data(), nullptr) < 0)
833 throw InternalErr (__FILE__, __LINE__, "Fail to select the hyperslab for reference dataset.");
834
835
836 mem_space_id = H5Screate_simple(d_num_dim,hcount.data(),nullptr);
837 if(mem_space_id < 0)
838 throw InternalErr(__FILE__, __LINE__, "Fail to obtain reference dataset memory space.");
839
840 if(H5Dread(dset_id,H5T_STD_REF,mem_space_id,file_space_id,H5P_DEFAULT,&rbuf[0])<0)
841 throw InternalErr(__FILE__, __LINE__, "Fail to read hyperslab reference dataset.");
842
843 H5Sclose(mem_space_id);
844 H5Sclose(file_space_id);
845
846 // Now we need to retrieve the reference info. fot the nelms elements.
847 vector<string> v_str;
848
849 H5R_type_t ref_type = H5Rget_type((const H5R_ref_t *)&rbuf[0]);
850
851 // The referenced objects can only be either objects or dataset regions.
852 if(ref_type != H5R_OBJECT2 && ref_type !=H5R_DATASET_REGION2)
853 throw InternalErr(__FILE__, __LINE__, "Unsupported reference: neither object nor region references");
854
855 for (int i = 0; i < nelms; i++) {
856
857 hid_t obj_id = H5Ropen_object((H5R_ref_t *)&rbuf[i], H5P_DEFAULT, H5P_DEFAULT);
858 if(obj_id < 0)
859 throw InternalErr(__FILE__, __LINE__, "Cannot open the object the reference points to");
860
861 vector<char> objname;
862 ssize_t objnamelen = -1;
863 if ((objnamelen= H5Iget_name(obj_id,nullptr,0))<=0) {
864 H5Oclose(obj_id);
865 throw InternalErr(__FILE__, __LINE__, "Cannot obtain the name length of the object the reference points to");
866 }
867 objname.resize(objnamelen+1);
868 if ((objnamelen= H5Iget_name(obj_id,objname.data(),objnamelen+1))<=0) {
869 H5Oclose(obj_id);
870 throw InternalErr(__FILE__, __LINE__, "Cannot obtain the name length of the object the reference points to");
871 }
872
873 string objname_str = string(objname.begin(),objname.end());
874 string trim_objname = objname_str.substr(0,objnamelen);
875
876 // For object references, we just need to save the object full path.
877 if(ref_type == H5R_OBJECT2)
878 v_str.push_back(trim_objname);
879 else {// Must be region reference.
880 H5O_type_t obj_type;
881 if(H5Rget_obj_type3((H5R_ref_t *)&rbuf[i], H5P_DEFAULT, &obj_type) < 0){
882 H5Oclose(obj_id);
883 throw InternalErr(__FILE__, __LINE__, "H5Rget_obj_type3() failed.");
884 }
885 if(obj_type != H5O_TYPE_DATASET) {
886 H5Oclose(obj_id);
887 throw InternalErr(__FILE__, __LINE__, "Region reference must point to a dataset.");
888 }
889 hid_t region_space_id = H5Ropen_region(&rbuf[i],H5P_DEFAULT,H5P_DEFAULT);
890 if (region_space_id < 0) {
891 H5Oclose(obj_id);
892 throw InternalErr(__FILE__, __LINE__, "Cannot obtain the space ID the reference points to");
893 }
894
895 int ndim = H5Sget_simple_extent_ndims(region_space_id);
896 if (ndim < 0) {
897 H5Sclose(region_space_id);
898 H5Oclose(obj_id);
899 throw InternalErr(__FILE__, __LINE__, "H5Sget_simple_extent_ndims() failed.");
900 }
901
902 string expression;
903 switch (H5Sget_select_type(region_space_id)) {
904
905 case H5S_SEL_NONE:
906 BESDEBUG("h5", "=read() None selected." << endl);
907 break;
908
909 case H5S_SEL_POINTS: {
910 BESDEBUG("h5", "=read() Points selected." << endl);
911 hssize_t npoints = H5Sget_select_npoints(region_space_id);
912 if (npoints < 0) {
913 H5Sclose(region_space_id);
914 H5Oclose(obj_id);
915 throw InternalErr(__FILE__, __LINE__,
916 "Cannot determine number of elements in the dataspace selection");
917 }
918
919 BESDEBUG("h5", "=read() npoints are " << npoints
920 << endl);
921 vector<hsize_t> buf(npoints * ndim);
922 if (H5Sget_select_elem_pointlist(region_space_id, 0, npoints, buf.data()) < 0) {
923 H5Sclose(region_space_id);
924 H5Oclose(obj_id);
925 throw InternalErr(__FILE__, __LINE__, "H5Sget_select_elem_pointlist() failed.");
926 }
927
928#if 0
929 for (int j = 0; j < npoints * ndim; j++) {
930 "h5", "=read() npoints buf[0] =" << buf[j] <<endl;
931 }
932#endif
933
934 for (int j = 0; j < (int) npoints; j++) {
935 // Name of the dataset.
936 expression.append(trim_objname);
937 for (int k = 0; k < ndim; k++) {
938 ostringstream oss;
939 oss << "[" << (int) buf[j * ndim + k] << "]";
940 expression.append(oss.str());
941 }
942 if (j != (int) (npoints - 1)) {
943 expression.append(",");
944 }
945 }
946 v_str.push_back(expression);
947
948 break;
949 }
950 case H5S_SEL_HYPERSLABS: {
951 vector<hsize_t> start(ndim);
952 vector<hsize_t> end(ndim);
953 vector<hsize_t>stride(ndim);
954 vector<hsize_t>s_count(ndim);
955 vector<hsize_t>block(ndim);
956
957 BESDEBUG("h5", "=read() Slabs selected." << endl);
958 BESDEBUG("h5", "=read() nblock is " <<
959 H5Sget_select_hyper_nblocks(region_space_id) << endl);
960
961 if (H5Sget_regular_hyperslab(region_space_id, start.data(), stride.data(), s_count.data(), block.data()) < 0) {
962 H5Sclose(region_space_id);
963 H5Oclose(obj_id);
964 throw InternalErr(__FILE__, __LINE__, "H5Sget_regular_hyperslab() failed.");
965 }
966
967 expression.append(trim_objname);
968 for (int j = 0; j < ndim; j++) {
969 ostringstream oss;
970 BESDEBUG("h5", "start " << start[j]
971 << "stride "<<stride[j]
972 << "count "<< s_count[j]
973 << "block "<< block[j]
974 <<endl);
975
976 // Map from HDF5's start,stride,count,block to DAP's start,stride,end.
977 end[j] = start[j] + stride[j]*(s_count[j]-1)+(block[j]-1);
978 BESDEBUG("h5", "=read() start is " << start[j]
979 << "=read() end is " << end[j] << endl);
980 oss << "[" << start[j] << ":" << stride[j] << ":" << end[j] << "]";
981 expression.append(oss.str());
982 BESDEBUG("h5", "=read() expression is "
983 << expression << endl)
984 ;
985 }
986 v_str.push_back(expression);
987 // Constraint expression. [start:stride:end]
988 break;
989 }
990 case H5S_SEL_ALL:
991 BESDEBUG("h5", "=read() All selected." << endl);
992 break;
993
994 default:
995 BESDEBUG("h5", "Unknown space type." << endl);
996 break;
997 }
998 H5Sclose(region_space_id);
999 }
1000 H5Oclose(obj_id);
1001 }
1002 for (int i = 0; i<nelms; i++)
1003 H5Rdestroy(&rbuf[i]);
1004 delete[] rbuf;
1005 H5Sclose(mem_space_id);
1006 H5Sclose(file_space_id);
1007 set_value(v_str.data(), nelms);
1008 return false;
1009 }
1010 catch (...) {
1011 if(rbuf!= nullptr)
1012 delete[] rbuf;
1013 H5Sclose(mem_space_id);
1014 H5Sclose(file_space_id);
1015 throw;
1016 }
1017#endif
1018}
1019
1020 // Unused code, keep it for a while.
1021 // Decide not check if the datatype is reference or not. This should be done by URL.
1022#if 0
1023 bool is_new_ref = false;
1024 bool is_obj_ref = false;
1025 bool is_reg_ref = false;
1026
1027 htri_t ref_ret = H5Tequal(dtype_id, H5T_STD_REF);
1028
1029 // Check if this is a new reference(>= HDF5 1.12).
1030 if(ref_ret < 0)
1031 throw InternalErr(__FILE__, __LINE__, "H5Tequal() failed to compare H5T_STD_REF");
1032 else if (ref_ret > 0)
1033 is_new_ref = true;
1034
1035 if(false == is_new_ref) {
1036
1037 ref_ret = H5Tequal(dtype_id,H5T_STD_REF_OBJ);
1038 // Check if this is the old object reference(< HDF5 1.12).
1039 if(ref_ret < 0)
1040 throw InternalErr(__FILE__, __LINE__, "H5Tequal() failed to compare H5T_STD_REF_OBJ");
1041 else if (ref_ret > 0)
1042 is_obj_ref = true;
1043
1044 if(ref_ret == 0) {
1045 // Check if this is the old region reference(< HDF5 1.12)
1046 ref_ret = H5Tequal(dtype_id,H5T_STD_REF_DSETREG);
1047 if(ref_ret < 0)
1048 throw InternalErr(__FILE__, __LINE__, "H5Tequal() failed to compare H5T_STD_REF_DSETREG");
1049 else if (ref_ret > 0)
1050 is_reg_ref = true;
1051 }
1052 }
1053 else {
1054 H5R_ref_t ref_type;
1055 ......
1056 }
1057 if(is_obj_ref == false && is_reg_ref == false)
1058 throw InternalErr(__FILE__, __LINE__, "datatype must be either object ref. or region ref.");
1059#endif
1060
1061
1062void HDF5Array::m_intern_plain_array_data(char *convbuf,hid_t memtype)
1063{
1064 if (check_h5str(memtype)) {
1065 vector<string> v_str(d_num_elm);
1066 size_t elesize = H5Tget_size(memtype);
1067 if (elesize == 0) {
1068 throw InternalErr(__FILE__, __LINE__, "H5Tget_size() failed.");
1069 }
1070 vector<char> strbuf(elesize + 1);
1071 BESDEBUG("h5", "=read()<check_h5str() element size=" << elesize
1072 << " d_num_elm=" << d_num_elm << endl);
1073
1074 for (int strindex = 0; strindex < d_num_elm; strindex++) {
1075 get_strdata(strindex, convbuf, strbuf.data(), (int)elesize);
1076 BESDEBUG("h5", "=read()<get_strdata() strbuf=" << strbuf.data() << endl);
1077 v_str[strindex] = strbuf.data();
1078 }
1079 set_read_p(true);
1080 val2buf((void *) v_str.data());
1081 }
1082 else {
1083 set_read_p(true);
1084 val2buf((void *) convbuf);
1085 }
1086}
1087
1088
1089bool HDF5Array::do_h5_array_type_read(hid_t dsetid, hid_t memb_id,vector<char>&values,bool has_values,int values_offset,
1090 int at_nelms,int* at_offset,int* at_count, int* at_step){
1091 //1. Call do array first(datatype must be derived) and the value must be set. We don't support Array datatype
1092 // unless it is inside a compound datatype
1093 if(has_values != true)
1094 throw InternalErr (__FILE__, __LINE__, "Only support the retrieval of HDF5 Array datatype values from the parent compound datatype read.");
1095
1096 hid_t at_base_type = H5Tget_super(memb_id);
1097 if(at_base_type < 0) {
1098 throw InternalErr (__FILE__, __LINE__, "Fail to obtain the basetype of the array datatype.");
1099 }
1100
1101 // memb_id, obtain the number of dimensions
1102 int at_ndims = H5Tget_array_ndims(memb_id);
1103 if(at_ndims <= 0) {
1104 H5Tclose(at_base_type);
1105 throw InternalErr (__FILE__, __LINE__, "Fail to obtain number of dimensions of the array datatype.");
1106 }
1107
1108 vector<hsize_t>at_dims_h(at_ndims,0);
1109
1110 // Obtain the number of elements for each dims
1111 if(H5Tget_array_dims(memb_id,at_dims_h.data())<0) {
1112 H5Tclose(at_base_type);
1113 throw InternalErr (__FILE__, __LINE__, "Fail to obtain dimensions of the array datatype.");
1114 }
1115 vector<int>at_dims(at_ndims,0);
1116 for(int i = 0;i<at_ndims;i++) {
1117 at_dims[i] = (int)at_dims_h[i];
1118 }
1119 int at_total_nelms = 1;
1120 for (int i = 0; i <at_ndims; i++)
1121 at_total_nelms = at_total_nelms*at_dims[i];
1122
1123 H5T_class_t array_cls = H5Tget_class(at_base_type);
1124 if(H5T_NO_CLASS == array_cls) {
1125 H5Tclose(at_base_type);
1126 throw InternalErr (__FILE__, __LINE__, "Fail to obtain the datatype class of the array base type.");
1127 }
1128
1129 size_t at_base_type_size = H5Tget_size(at_base_type);
1130 if(0 == at_base_type_size){
1131 H5Tclose(at_base_type);
1132 throw InternalErr (__FILE__, __LINE__, "Fail to obtain the size of the array base type.");
1133 }
1134
1135 // H5 Array type, the basetype is COMPOUND.
1136 if(H5T_COMPOUND == array_cls) {
1137
1138 // These vectors are used to handle subset of array datatype
1139 vector<int>at_end(at_ndims,0);
1140 vector<int>at_pos(at_ndims,0);
1141 for (int i = 0; i< at_ndims; i++){
1142 at_pos[i] = at_offset[i];
1143 at_end[i] = at_offset[i] + (at_count[i] -1)*at_step[i];
1144 }
1145
1146 int at_orig_index = INDEX_nD_TO_1D(at_dims,at_pos);
1147
1148 // To read the array of compound (structure) in DAP, one must read one element each. set_vec is used afterwards.
1149 for (int array_index = 0; array_index <at_nelms; array_index++) {
1150
1151 // The basetype of the array datatype is compound,-- check if the following line is valid.
1152 HDF5Structure *h5s = dynamic_cast<HDF5Structure*>(var()->ptr_duplicate());
1153 hid_t child_memb_id;
1154 H5T_class_t child_memb_cls;
1155 int child_nmembs;
1156 size_t child_memb_offset;
1157
1158 if((child_nmembs = H5Tget_nmembers(at_base_type)) < 0) {
1159 H5Tclose(at_base_type);
1160 delete h5s;
1161 throw InternalErr (__FILE__, __LINE__, "Fail to obtain number of HDF5 compound datatype.");
1162 }
1163
1164 for(unsigned child_u = 0; child_u < (unsigned)child_nmembs; child_u++) {
1165
1166 // Get member type ID
1167 if((child_memb_id = H5Tget_member_type(at_base_type, child_u)) < 0) {
1168 H5Tclose(at_base_type);
1169 delete h5s;
1170 throw InternalErr (__FILE__, __LINE__, "Fail to obtain the datatype of an HDF5 compound datatype member.");
1171 }
1172
1173 // Get member type class
1174 if((child_memb_cls = H5Tget_member_class (at_base_type, child_u)) < 0) {
1175 H5Tclose(child_memb_id);
1176 H5Tclose(at_base_type);
1177 delete h5s;
1178 throw InternalErr (__FILE__, __LINE__, "Fail to obtain the datatype class of an HDF5 compound datatype member.");
1179 }
1180
1181 // Get member offset
1182 child_memb_offset= H5Tget_member_offset(at_base_type,child_u);
1183
1184 // Get member name
1185 char *child_memb_name = H5Tget_member_name(at_base_type,child_u);
1186 if(child_memb_name == nullptr) {
1187 H5Tclose(child_memb_id);
1188 H5Tclose(at_base_type);
1189 delete h5s;
1190 throw InternalErr (__FILE__, __LINE__, "Fail to obtain the name of an HDF5 compound datatype member.");
1191 }
1192
1193 BaseType *field = h5s->var(child_memb_name);
1194 if (child_memb_cls == H5T_COMPOUND) {
1195
1196 HDF5Structure &memb_h5s = dynamic_cast<HDF5Structure&>(*field);
1197 //
1198 // Call structure read when reading the whole array. sa1{sa2[100]}
1199 // sa2[100] is an array datatype.
1200 // If reading the whole buffer, just add the buffer.
1201 if(at_total_nelms == at_nelms) {
1202 memb_h5s.do_structure_read(dsetid,child_memb_id, values,has_values,values_offset+at_base_type_size*array_index+child_memb_offset);
1203 }
1204 // Subset of sa2, sa2[10:100:2]; sa2[100] is an array datatype. The whole array sa2[100] is to be read into somewhere in buffer values.
1205 else {// The subset should be considered. adjust memb_offset+values_offset+???,make sure only the subset is selected.
1206 // at_total_nelms is 100 but at_nelms is (100-10)/2+1=46. The starting point of the whole array is values+memb_offset_values_offset
1207 // When the datatype is structure, we have to obtain the index one by one.
1208
1209 memb_h5s.do_structure_read(dsetid, child_memb_id, values,has_values,values_offset+at_base_type_size*at_orig_index+child_memb_offset);
1210
1211 }
1212
1213 }
1214 else if(child_memb_cls == H5T_ARRAY) {
1215
1216 // memb_id, obtain the number of dimensions
1217 int child_at_ndims = H5Tget_array_ndims(child_memb_id);
1218 if(child_at_ndims <= 0) {
1219 H5Tclose(at_base_type);
1220 H5Tclose(child_memb_id);
1221 H5free_memory(child_memb_name);
1222 delete h5s;
1223 throw InternalErr (__FILE__, __LINE__, "Fail to obtain number of dimensions of the array datatype.");
1224
1225 }
1226
1227 HDF5Array &h5_array_type = dynamic_cast<HDF5Array&>(*field);
1228 vector<int> child_at_offset(child_at_ndims,0);
1229 vector<int> child_at_count(child_at_ndims,0);
1230 vector<int> child_at_step(child_at_ndims,0);
1231
1232 int child_at_nelms = h5_array_type.format_constraint(child_at_offset.data(),child_at_step.data(),child_at_count.data());
1233
1234 if(at_total_nelms == at_nelms) {
1235 h5_array_type.do_h5_array_type_read(dsetid,child_memb_id,values,has_values,child_memb_offset+values_offset+at_base_type_size*array_index,
1236 child_at_nelms,child_at_offset.data(),child_at_count.data(),child_at_step.data());
1237 }
1238 else {// Adjust memb_offset+values_offset, basically change at_base_type_size*array_index
1239 h5_array_type.do_h5_array_type_read(dsetid,child_memb_id,values,has_values,child_memb_offset+values_offset+at_base_type_size*at_orig_index,
1240 child_at_nelms,child_at_offset.data(),child_at_count.data(),child_at_step.data());
1241
1242 }
1243 }
1244
1245 else if(H5T_INTEGER == child_memb_cls || H5T_FLOAT == child_memb_cls){
1246
1247 int number_index =((at_total_nelms == at_nelms)?array_index:at_orig_index);
1248 if(true == promote_char_to_short(child_memb_cls,child_memb_id)) {
1249 void *src = (void*)(values.data() + (number_index*at_base_type_size) + values_offset +child_memb_offset);
1250 char val_int8;
1251 memcpy(&val_int8,src,1);
1252 short val_short=(short)val_int8;
1253 field->val2buf(&val_short);
1254 }
1255 else
1256 field->val2buf(values.data() + (number_index * at_base_type_size) + values_offset+child_memb_offset);
1257
1258
1259 }
1260 else if(H5T_STRING == child_memb_cls){
1261
1262 int string_index =((at_total_nelms == at_nelms)?array_index:at_orig_index);
1263
1264 // distinguish between variable length and fixed length
1265 if(true == H5Tis_variable_str(child_memb_id)) {
1266
1267 // Need to check if the size of variable length array type is right in HDF5 lib.
1268 void *src = (void*)(values.data()+(string_index *at_base_type_size)+values_offset+child_memb_offset);
1269 string final_str;
1270 char*temp_bp =(char*)src;
1271 get_vlen_str_data(temp_bp,final_str);
1272 field->val2buf(&final_str[0]);
1273#if 0
1274 field->set_value(final_str);
1275#endif
1276 }
1277 else {// Obtain string
1278 void *src = (void*)(values.data()+(string_index *at_base_type_size)+values_offset+child_memb_offset);
1279 vector<char> str_val;
1280 size_t memb_size = H5Tget_size(child_memb_id);
1281 if (memb_size == 0) {
1282 H5Tclose(child_memb_id);
1283 H5Tclose(at_base_type);
1284 H5free_memory(child_memb_name);
1285 delete h5s;
1286 throw InternalErr (__FILE__, __LINE__,"Fail to obtain the size of HDF5 compound datatype.");
1287 }
1288 str_val.resize(memb_size);
1289 memcpy(str_val.data(),src,memb_size);
1290 field->val2buf(str_val.data());
1291
1292 }
1293 }
1294 else {
1295 H5Tclose(child_memb_id);
1296 H5Tclose(at_base_type);
1297 H5free_memory(child_memb_name);
1298 delete h5s;
1299 throw InternalErr (__FILE__, __LINE__, "Unsupported datatype class for the array base type.");
1300
1301
1302 }
1303 field->set_read_p(true);
1304 H5free_memory(child_memb_name);
1305 H5Tclose(child_memb_id);
1306
1307 } // end "for ( child_u = 0)"
1308 h5s->set_read_p(true);
1309
1310 // Save the value of this element to DAP structure.
1311 set_vec(array_index,h5s);
1312 delete h5s;
1313
1314 vector<int>at_offsetv(at_pos.size(),0);
1315 vector<int>at_stepv(at_pos.size(),0);
1316 for (unsigned int at_index = 0; at_index<at_pos.size();at_index++){
1317 at_offsetv[at_index] = at_offset[at_index];
1318 at_stepv[at_index] = at_step[at_index];
1319 }
1320 //obtain the next position of the selected point based on the offset,end and step.
1321 obtain_next_pos(at_pos,at_offsetv,at_end,at_stepv,(int)(at_pos.size()));
1322 at_orig_index = INDEX_nD_TO_1D(at_dims,at_pos);
1323 }// end for "(array_index = 0) for array (compound)datatype"
1324
1325 // Need to check the usage of set_read_p(true);
1326#if 0
1327 //set_read_p(true);
1328#endif
1329 }
1330 else if(H5T_INTEGER == array_cls|| H5T_FLOAT == array_cls) {
1331
1332 // If no subset for the array datatype, just read the whole buffer.
1333 if(at_total_nelms == at_nelms) {
1334
1335 // For DAP2 char should be mapped to short
1336 if( true == promote_char_to_short(array_cls ,at_base_type)) {
1337 vector<char> val_int8;
1338 val_int8.resize(at_nelms);
1339 void*src = (void*)(values.data() +values_offset);
1340 memcpy(val_int8.data(),src,at_nelms);
1341
1342 vector<short> val_short;
1343 for (int i = 0; i<at_nelms; i++)
1344 val_short[i] = (short)val_int8[i];
1345
1346 val2buf(val_short.data());
1347
1348 }
1349 else // short cut for others
1350 val2buf(values.data() + values_offset);
1351
1352 }
1353 else { // Adjust the value for the subset of the array datatype
1354
1355 // Obtain the correponding DAP type of the HDF5 data type
1356 string dap_type = get_dap_type(at_base_type,is_dap4());
1357
1358 // The total array type data is read.
1359 void*src = (void*)(values.data() + values_offset);
1360
1361 // set the original position to the starting point
1362 vector<int>at_pos(at_ndims,0);
1363 for (int i = 0; i< at_ndims; i++)
1364 at_pos[i] = at_offset[i];
1365
1366 if( BYTE == dap_type) {
1367
1368 vector<unsigned char>total_val;
1369 total_val.resize(at_total_nelms);
1370 memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1371
1372 vector<unsigned char>final_val;
1373 subset<unsigned char>(
1374 total_val.data(),
1375 at_ndims,
1376 at_dims,
1377 at_offset,
1378 at_step,
1379 at_count,
1380 &final_val,
1381 at_pos,
1382 0
1383 );
1384
1385 set_value(final_val.data(),at_nelms);
1386
1387
1388 }
1389 else if( INT16 == dap_type) {
1390
1391 // promote char to short,DAP2 doesn't have "char" type
1392 if(true == promote_char_to_short(array_cls,at_base_type)) {
1393 vector<char>total_val;
1394 total_val.resize(at_total_nelms);
1395 memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1396
1397 vector<char>final_val;
1398 subset<char>(
1399 total_val.data(),
1400 at_ndims,
1401 at_dims,
1402 at_offset,
1403 at_step,
1404 at_count,
1405 &final_val,
1406 at_pos,
1407 0
1408 );
1409
1410 vector<short> final_val_short;
1411 final_val_short.resize(at_nelms);
1412 for(int i = 0; i<at_nelms; i++)
1413 final_val_short[i] = final_val[i];
1414
1415 val2buf(final_val_short.data());
1416
1417 }
1418 else {// short
1419
1420 vector<short>total_val;
1421 total_val.resize(at_total_nelms);
1422 memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1423
1424 vector<short>final_val;
1425 subset<short>(
1426 total_val.data(),
1427 at_ndims,
1428 at_dims,
1429 at_offset,
1430 at_step,
1431 at_count,
1432 &final_val,
1433 at_pos,
1434 0
1435 );
1436
1437 val2buf(final_val.data());
1438
1439 }
1440 }
1441 else if( UINT16 == dap_type) {
1442 vector<unsigned short>total_val;
1443 total_val.resize(at_total_nelms);
1444 memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1445
1446 vector<unsigned short>final_val;
1447 subset<unsigned short>(
1448 total_val.data(),
1449 at_ndims,
1450 at_dims,
1451 at_offset,
1452 at_step,
1453 at_count,
1454 &final_val,
1455 at_pos,
1456 0
1457 );
1458
1459 val2buf(final_val.data());
1460
1461 }
1462 else if(UINT32 == dap_type) {
1463 vector<unsigned int>total_val;
1464 total_val.resize(at_total_nelms);
1465 memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1466
1467 vector<unsigned int>final_val;
1468 subset<unsigned int>(
1469 total_val.data(),
1470 at_ndims,
1471 at_dims,
1472 at_offset,
1473 at_step,
1474 at_count,
1475 &final_val,
1476 at_pos,
1477 0
1478 );
1479 val2buf(final_val.data());
1480
1481
1482 }
1483 else if(INT32 == dap_type) {
1484 vector<int>total_val;
1485 total_val.resize(at_total_nelms);
1486 memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1487
1488 vector<int>final_val;
1489 subset<int>(
1490 total_val.data(),
1491 at_ndims,
1492 at_dims,
1493 at_offset,
1494 at_step,
1495 at_count,
1496 &final_val,
1497 at_pos,
1498 0
1499 );
1500
1501 val2buf(final_val.data());
1502
1503 }
1504 else if(FLOAT32 == dap_type) {
1505 vector<float>total_val;
1506 total_val.resize(at_total_nelms);
1507 memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1508
1509 vector<float>final_val;
1510 subset<float>(
1511 total_val.data(),
1512 at_ndims,
1513 at_dims,
1514 at_offset,
1515 at_step,
1516 at_count,
1517 &final_val,
1518 at_pos,
1519 0
1520 );
1521
1522 val2buf(final_val.data());
1523
1524 }
1525 else if(FLOAT64 == dap_type) {
1526 vector<double>total_val;
1527 total_val.resize(at_total_nelms);
1528 memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1529
1530 vector<double>final_val;
1531 subset<double>(
1532 total_val.data(),
1533 at_ndims,
1534 at_dims,
1535 at_offset,
1536 at_step,
1537 at_count,
1538 &final_val,
1539 at_pos,
1540 0
1541 );
1542#if 0
1543 //field->val2buf(final_val.data());
1544#endif
1545 val2buf(final_val.data());
1546
1547 }
1548 else {
1549 H5Tclose(at_base_type);
1550 throw InternalErr (__FILE__, __LINE__,
1551 "Non-supported integer or float datatypes");
1552 }
1553
1554 }
1555 }
1556 else if(H5T_STRING == array_cls) {
1557
1558 // set the original position to the starting point
1559 vector<int>at_pos(at_ndims,0);
1560 for (int i = 0; i< at_ndims; i++)
1561 at_pos[i] = at_offset[i];
1562
1563 vector<string>total_strval;
1564 total_strval.resize(at_total_nelms);
1565
1566 if(true == H5Tis_variable_str(at_base_type)) {
1567 void *src = (void*)(values.data()+values_offset);
1568 char*temp_bp =(char*)src;
1569 for(int i = 0;i <at_total_nelms; i++){
1570 string tempstrval;
1571 get_vlen_str_data(temp_bp,tempstrval);
1572 total_strval[i] = tempstrval;
1573 temp_bp += at_base_type_size;
1574 }
1575 if(at_total_nelms == at_nelms) {
1576#if 0
1577 //field->set_value(total_strval,at_total_nelms);
1578#endif
1579 set_value(total_strval,at_total_nelms);
1580 }
1581 else {// obtain subset for variable-length string.
1582#if 0
1583 //field->val2buf(values.data() + values_offset);
1584#endif
1585 vector<string>final_val;
1586 subset<string>(
1587 total_strval.data(),
1588 at_ndims,
1589 at_dims,
1590 at_offset,
1591 at_step,
1592 at_count,
1593 &final_val,
1594 at_pos,
1595 0
1596 );
1597
1598 set_value(final_val,at_nelms);
1599
1600 }
1601
1602 }
1603 else {// For fixed-size string.
1604 void *src = (void*)(values.data()+values_offset);
1605 for(int i = 0; i <at_total_nelms; i++)
1606 total_strval[i].resize(at_base_type_size);
1607
1608 vector<char> str_val;
1609 str_val.resize(at_total_nelms*at_base_type_size);
1610 memcpy((void*)str_val.data(),src,at_total_nelms*at_base_type_size);
1611 string total_in_one_string(str_val.begin(),str_val.end());
1612 for(int i = 0; i<at_total_nelms;i++)
1613 total_strval[i] = total_in_one_string.substr(i*at_base_type_size,at_base_type_size);
1614
1615 if(at_total_nelms == at_nelms)
1616 set_value(total_strval,at_total_nelms);
1617 else {
1618 vector<string>final_val;
1619 subset<string>(
1620 total_strval.data(),
1621 at_ndims,
1622 at_dims,
1623 at_offset,
1624 at_step,
1625 at_count,
1626 &final_val,
1627 at_pos,
1628 0
1629 );
1630 set_value(final_val,at_nelms);
1631
1632 }
1633 }
1634
1635 }
1636 else {
1637 H5Tclose(at_base_type);
1638 throw InternalErr (__FILE__, __LINE__,
1639 "Only support the field of compound datatype when the field type class is integer, float, string, array or compound..");
1640
1641 }
1642
1643 H5Tclose(at_base_type);
1644
1645 return true;
1646}
1647
1649inline int
1650HDF5Array::INDEX_nD_TO_1D (const std::vector < int > &dims,
1651 const std::vector < int > &pos)
1652{
1653 //
1654 // "int a[10][20][30] // & a[1][2][3] == a + (20*30+1 + 30*2 + 1 *3)"
1655 // "int b[10][2]; // &b[1][2] == b + (20*1 + 2);"
1656 //
1657 assert (dims.size () == pos.size ());
1658 int sum = 0;
1659 int start = 1;
1660
1661 for (unsigned int p = 0; p < pos.size (); p++) {
1662 int m = 1;
1663
1664 for (unsigned int j = start; j < dims.size (); j++)
1665 m *= dims[j];
1666 sum += m * pos[p];
1667 start++;
1668 }
1669 return sum;
1670}
1671
1672// Obtain the dimension index of the next pos. of the point based on the offset, step and end
1673bool HDF5Array::obtain_next_pos(vector<int>& pos, vector<int>&start,vector<int>&end,vector<int>&step,int rank_change) {
1674
1675 if((pos[rank_change-1] + step[rank_change-1])<=end[rank_change-1]) {
1676 pos[rank_change-1] = pos[rank_change-1] + step[rank_change-1];
1677 return true;
1678 }
1679 else {
1680 if( 1 == rank_change)
1681 return false;
1682 pos[rank_change-1] = start[rank_change-1];
1683 obtain_next_pos(pos,start,end,step,rank_change-1);
1684 }
1685 return true;
1686}
1687
1689//
1690// \param input Input variable
1691// \param dim dimension info of the input
1692// \param start start indexes of each dim
1693// \param stride stride of each dim
1694// \param edge count of each dim
1695// \param poutput output variable
1696// \parrm index dimension index
1697// \return 0 if successful. -1 otherwise.
1698//
1699template<typename T>
1700int HDF5Array::subset(
1701 const T input[],
1702 int rank,
1703 vector<int> & dim,
1704 int start[],
1705 int stride[],
1706 int edge[],
1707 std::vector<T> *poutput,
1708 vector<int>& pos,
1709 int index)
1710{
1711 for(int k=0; k<edge[index]; k++)
1712 {
1713 pos[index] = start[index] + k*stride[index];
1714 if(index+1<rank)
1715 subset(input, rank, dim, start, stride, edge, poutput,pos,index+1);
1716 if(index==rank-1)
1717 {
1718 poutput->push_back(input[INDEX_nD_TO_1D( dim, pos)]);
1719 }
1720 } // end of for
1721 return 0;
1722} // end of template<typename T> static int subset
1723
1724
1725// public functions to set all parameters needed in read function.
1726
1727
1728void HDF5Array::set_memneed(size_t need) {
1729 d_memneed = need;
1730}
1731
1732void HDF5Array::set_numdim(int ndims) {
1733 d_num_dim = ndims;
1734}
1735
1736void HDF5Array::set_numelm(int nelms) {
1737 d_num_elm = nelms;
1738}
1739
1740// TODO: This routine is not used anymore. May delete it in the future.
1741#if 0
1742hid_t HDF5Array::mkstr(int size, H5T_str_t pad)
1743{
1744
1745 hid_t str_type;
1746
1747 if ((str_type = H5Tcopy(H5T_C_S1)) < 0)
1748 return -1;
1749 if (H5Tset_size(str_type, (size_t) size) < 0)
1750 return -1;
1751 if (H5Tset_strpad(str_type, pad) < 0)
1752 return -1;
1753
1754 return str_type;
1755}
1756#endif
1757
1758// We don't inherit libdap Array Class's transform_to_dap4 method since CF option is still using it.
1759BaseType* HDF5Array::h5dims_transform_to_dap4(D4Group *grp,const vector<string> &dimpath) {
1760
1761 BESDEBUG("h5", "<h5dims_transform_to_dap4" << endl);
1762
1763 if(grp == nullptr)
1764 return nullptr;
1765
1766 Array *dest = static_cast<HDF5Array*>(ptr_duplicate());
1767
1768 // If there is just a size, don't make
1769 // a D4Dimension (In DAP4 you cannot share a dimension unless it has
1770 // a name). jhrg 3/18/14
1771
1772 int k = 0;
1773 for (Array::Dim_iter d = dest->dim_begin(), e = dest->dim_end(); d != e; ++d) {
1774
1775 if (false == (*d).name.empty()) {
1776 BESDEBUG("h5", "<coming to the dimension loop, has name " << (*d).name<<endl);
1777 BESDEBUG("h5", "<coming to the dimension loop, has dimpath " << dimpath[k] <<endl);
1778 BESDEBUG("h5", "<coming to the dimension loop, has dimpath group " << dimpath[k].substr(0,dimpath[k].find_last_of("/")+1) <<endl);
1779
1780 D4Group *temp_grp = grp;
1781 D4Dimension *d4_dim = nullptr;
1782 bool is_dim_nonc4_grp = false;
1783
1784 while(temp_grp) {
1785
1786 BESDEBUG("h5", "<coming to the group has name " << temp_grp->name()<<endl);
1787 BESDEBUG("h5", "<coming to the group has fullpath " << temp_grp->FQN()<<endl);
1788
1789 //Obtain all the dimensions of this group.
1790 D4Dimensions *temp_dims = temp_grp->dims();
1791
1792 // Check if this dimension is defined in this group
1793 d4_dim = temp_dims->find_dim((*d).name);
1794
1795 // Need the full path of the dimension name
1796 string d4_dim_path = dimpath[k].substr(0,dimpath[k].find_last_of("/")+1);
1797 BESDEBUG("h5", "d4_dim_path is " << d4_dim_path<<endl);
1798
1799 bool ancestor_grp = false;
1800
1801 // If the dim_path is within this group or its ancestor, this is valid.
1802 if(d4_dim_path.find(temp_grp->FQN())==0 || temp_grp->FQN().find(d4_dim_path)==0)
1803 ancestor_grp = true;
1804
1805 // If we find this dimension and the dimension is on the ancestral path,
1806 // this follows the netCDF-4/DAP4 dimension model, break.
1807 if(d4_dim && (temp_grp->FQN() == d4_dim_path)) {
1808 BESDEBUG("h5", "<FInd dimension name " << (*d).name<<endl);
1809 (*d).dim = d4_dim;
1810 is_dim_nonc4_grp = false;
1811 break;
1812 }
1813 // If the dimension name is not on the ancestral path, this
1814 // dimension must be on another path, mark it.
1815#if 0
1816 //else if( ancestor_grp == false && is_dim_nonc4_grp == false) {
1817#endif
1818 else if( ancestor_grp == false) {
1819 is_dim_nonc4_grp = true;
1820 break;
1821 }
1822 else
1823 d4_dim = nullptr;
1824
1825 if(temp_grp->get_parent())
1826 temp_grp = static_cast<D4Group*>(temp_grp->get_parent());
1827 else
1828 temp_grp = nullptr;
1829
1830 }
1831
1832 // Not find this dimension in any of the ancestor groups, add it to this group.
1833 // The following block is fine, but to avoid the complaint from sonarcloud.
1834 // Use a bool.
1835 if(true == is_dim_nonc4_grp) {
1836 string err= "The variable " + var_path +" has dimension ";
1837 err += dimpath[k] + ". This dimension is not under its ancestor or the current group.";
1838 err += " This is not supported.";
1839 delete dest;
1840 throw InternalErr(__FILE__,__LINE__,err);
1841 }
1842
1843 bool d4_dim_null = ((d4_dim==nullptr)?true:false);
1844 if(d4_dim_null == true) {
1845 d4_dim = new D4Dimension((*d).name, (*d).size);
1846 D4Dimensions * dims = grp->dims();
1847 BESDEBUG("h5", "<Just before adding D4 dimension to group" << endl);
1848 dims->add_dim_nocopy(d4_dim);
1849 (*d).dim = d4_dim;
1850 }
1851 }
1852 k++;
1853 }
1854
1855 dest->set_is_dap4(true);
1856
1857 return dest;
1858
1859}
A class for handling all types of array in HDF5 for the default option.
This class that translates HDF5 string into DAP string for the default option.
This class converts HDF5 compound type into DAP structure for the default option.
libdap::BaseType * ptr_duplicate() override
Definition: HDF5Array.cc:54
bool read() override
Reads HDF5 array data into local buffer.
Definition: HDF5Array.cc:109
void set_numdim(int ndims)
remembers number of dimensions of this array.
Definition: HDF5Array.cc:1732
HDF5Array(const std::string &n, const std::string &d, libdap::BaseType *v)
Constructor.
Definition: HDF5Array.cc:58
void set_numelm(int nelms)
remembers number of elements in this array.
Definition: HDF5Array.cc:1736
void set_memneed(size_t need)
remembers memory size needed.
Definition: HDF5Array.cc:1728
void get_data(hid_t dset, void *buf)
Definition: h5common.cc:50
void get_strdata(int strindex, char *allbuf, char *buf, int elesize)
Definition: h5common.cc:115
bool check_h5str(hid_t h5type)
Definition: h5get.cc:849
string get_dap_type(hid_t type, bool is_dap4)
Definition: h5get.cc:292
const int DODS_NAMELEN
Maximum length of variable or attribute name(default option only).
Definition: hdf5_handler.h:65