35#include "config_hdf5.h"
44#include <libdap/Error.h>
45#include <libdap/InternalErr.h>
63int HDF5Array::format_constraint(
int *offset,
int *step,
int *count) {
72 Dim_iter p = dim_begin();
74 while (p != dim_end()) {
76 int start = dimension_start(p,
true);
77 int stride = dimension_stride(p,
true);
78 int stop = dimension_stop(p,
true);
84 oss <<
"Array/Grid hyperslab start point "<< start <<
85 " is greater than stop point " << stop <<
".";
86 throw Error(malformed_expr, oss.str());
91 count[id] = ((stop - start) / stride) + 1;
95 "=format_constraint():"
96 <<
"id=" <<
id <<
" offset=" << offset[
id]
97 <<
" step=" << step[
id]
98 <<
" count=" << count[
id]
112 ">read() dataset=" << dataset()
113 <<
" dimension=" << d_num_dim
114 <<
" data_size=" << d_memneed <<
" length=" << length()
117 hid_t file_id = H5Fopen(dataset().c_str(),H5F_ACC_RDONLY,H5P_DEFAULT);
119 BESDEBUG(
"h5",
"after H5Fopen "<<endl);
120 BESDEBUG(
"h5",
"variable name is "<<name() <<endl);
121 BESDEBUG(
"h5",
"variable path is "<<var_path <<endl);
125 if(
true == is_dap4())
126 dset_id = H5Dopen2(file_id,var_path.c_str(),H5P_DEFAULT);
128 dset_id = H5Dopen2(file_id,name().c_str(),H5P_DEFAULT);
130 BESDEBUG(
"h5",
"after H5Dopen2 "<<endl);
133 hid_t dspace_id = H5Dget_space(dset_id);
137 throw InternalErr(__FILE__,__LINE__,
"Fail to obtain the dataspace .");
140 int num_dim = H5Sget_simple_extent_ndims(dspace_id);
145 throw InternalErr(__FILE__,__LINE__,
"Fail to obtain the datatype .");
151 hid_t dtype_id = H5Dget_type(dset_id);
155 throw InternalErr(__FILE__,__LINE__,
"Fail to obtain the datatype .");
159 vector<int> offset(d_num_dim);
160 vector<int> count(d_num_dim);
161 vector<int> step(d_num_dim);
162 int nelms = format_constraint(offset.data(), step.data(), count.data());
168 bool ret_ref =
false;
170 ret_ref = m_array_of_reference(dset_id,dtype_id);
187 do_array_read(dset_id,dtype_id,values,
false,0,nelms,offset.data(),count.data(),step.data());
203void HDF5Array::do_array_read(hid_t dset_id,hid_t dtype_id,vector<char>&values,
bool has_values,
int values_offset,
204 int nelms,
int* offset,
int* count,
int* step)
207 H5T_class_t tcls = H5Tget_class(dtype_id);
209 if(H5T_COMPOUND == tcls)
210 m_array_of_structure(dset_id,values,has_values,values_offset,nelms,offset,count,step);
211 else if(H5T_INTEGER == tcls || H5T_FLOAT == tcls || H5T_STRING == tcls)
212 m_array_of_atomic(dset_id,dtype_id,nelms,offset,count,step);
214 throw InternalErr(__FILE__,__LINE__,
"Fail to read the data for Unsupported datatype.");
219void HDF5Array:: m_array_of_atomic(hid_t dset_id, hid_t dtype_id,
220 int nelms,
int* offset,
int* count,
int* step)
224 if((memtype = H5Tget_native_type(dtype_id, H5T_DIR_ASCEND))<0) {
225 throw InternalErr (__FILE__, __LINE__,
"Fail to obtain memory datatype.");
229 if (H5Tis_variable_str(memtype) && H5Tget_class(memtype) == H5T_STRING) {
231 vector<hsize_t> hoffset;
232 vector<hsize_t>hcount;
233 vector<hsize_t>hstep;
234 hoffset.resize(d_num_dim);
235 hcount.resize(d_num_dim);
236 hstep.resize(d_num_dim);
237 for (
int i = 0; i <d_num_dim; i++) {
238 hoffset[i] = (hsize_t) offset[i];
239 hcount[i] = (hsize_t) count[i];
240 hstep[i] = (hsize_t) step[i];
243 vector<string>finstrval;
244 finstrval.resize(nelms);
246 read_vlen_string(dset_id, nelms, hoffset.data(), hstep.data(), hcount.data(),finstrval);
250 throw InternalErr(__FILE__,__LINE__,
"Fail to read variable-length string.");
252 set_value(finstrval,nelms);
258 if (nelms == d_num_elm) {
260 vector<char> convbuf(d_memneed);
261 get_data(dset_id, (
void *) convbuf.data());
264 if(
false == is_dap4()) {
265 if (1 == H5Tget_size(memtype) && H5T_SGN_2 == H5Tget_sign(memtype))
267 vector<short> convbuf2(nelms);
268 for (
int i = 0; i < nelms; i++) {
269 convbuf2[i] = (
signed char) (convbuf[i]);
270 BESDEBUG(
"h5",
"convbuf[" << i <<
"]="
271 << (
signed char)convbuf[i] << endl);
272 BESDEBUG(
"h5",
"convbuf2[" << i <<
"]="
273 << convbuf2[i] << endl)
277 m_intern_plain_array_data((
char*) convbuf2.data(),memtype);
280 m_intern_plain_array_data(convbuf.data(),memtype);
283 m_intern_plain_array_data(convbuf.data(),memtype);
286 size_t data_size = nelms * H5Tget_size(memtype);
287 if (data_size == 0) {
288 throw InternalErr(__FILE__, __LINE__,
"get_size failed");
290 vector<char> convbuf(data_size);
291 get_slabdata(dset_id, offset, step, count, d_num_dim, convbuf.data());
294 if(
false == is_dap4()){
295 if (1 == H5Tget_size(memtype) && H5T_SGN_2 == H5Tget_sign(memtype)) {
296 vector<short> convbuf2(data_size);
297 for (
int i = 0; i < (
int)data_size; i++) {
298 convbuf2[i] =
static_cast<signed char> (convbuf[i]);
300 m_intern_plain_array_data((
char*) convbuf2.data(),memtype);
303 m_intern_plain_array_data(convbuf.data(),memtype);
307 m_intern_plain_array_data(convbuf.data(),memtype);
319bool HDF5Array::m_array_of_structure(hid_t dsetid, vector<char>&values,
bool has_values,
int values_offset,
320 int nelms,
const int* offset,
const int* count,
const int* step) {
322 BESDEBUG(
"h5",
"=read() Array of Structure length=" << length() << endl);
329 if((dtypeid = H5Dget_type(dsetid)) < 0)
330 throw InternalErr (__FILE__, __LINE__,
"Cannot obtain the datatype.");
332 if((memtype = H5Tget_native_type(dtypeid, H5T_DIR_ASCEND))<0) {
334 throw InternalErr (__FILE__, __LINE__,
"Fail to obtain memory datatype.");
337 ty_size = H5Tget_size(memtype);
341 throw InternalErr (__FILE__, __LINE__,
"Fail to obtain the size of HDF5 compound datatype.");
344 if(
false == has_values) {
348 if ((dspace = H5Dget_space(dsetid))<0) {
351 throw InternalErr (__FILE__, __LINE__,
"Cannot obtain data space.");
354 d_num_dim = H5Sget_simple_extent_ndims(dspace);
359 throw InternalErr (__FILE__, __LINE__,
"Cannot obtain the number of dimensions of the data space.");
362 vector<hsize_t> hoffset;
363 vector<hsize_t>hcount;
364 vector<hsize_t>hstep;
365 hoffset.resize(d_num_dim);
366 hcount.resize(d_num_dim);
367 hstep.resize(d_num_dim);
368 for (
int i = 0; i <d_num_dim; i++) {
369 hoffset[i] = (hsize_t) offset[i];
370 hcount[i] = (hsize_t) count[i];
371 hstep[i] = (hsize_t) step[i];
374 if (H5Sselect_hyperslab(dspace, H5S_SELECT_SET,
375 hoffset.data(), hstep.data(),
376 hcount.data(),
nullptr) < 0) {
380 throw InternalErr (__FILE__, __LINE__,
"Cannot generate the hyperslab of the HDF5 dataset.");
383 mspace = H5Screate_simple(d_num_dim, hcount.data(),
nullptr);
388 throw InternalErr (__FILE__, __LINE__,
"Cannot create the memory space.");
391 values.resize(nelms*ty_size);
393 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,(
void*)values.data());
398 throw InternalErr (__FILE__, __LINE__,
"Fail to read the HDF5 compound datatype dataset.");
407 char* memb_name =
nullptr;
412 for (
int element = 0; element < nelms; ++element) {
415 H5T_class_t memb_cls = H5T_NO_CLASS;
417 size_t memb_offset = 0;
419 if((nmembs = H5Tget_nmembers(memtype)) < 0)
420 throw InternalErr (__FILE__, __LINE__,
"Fail to obtain number of HDF5 compound datatype.");
422 for(
unsigned int u = 0; u < (unsigned)nmembs; u++) {
425 if((memb_id = H5Tget_member_type(memtype, u)) < 0)
426 throw InternalErr (__FILE__, __LINE__,
"Fail to obtain the datatype of an HDF5 compound datatype member.");
429 if((memb_cls = H5Tget_member_class (memtype, u)) < 0)
430 throw InternalErr (__FILE__, __LINE__,
"Fail to obtain the datatype class of an HDF5 compound datatype member.");
435 memb_offset= H5Tget_member_offset(memtype,u);
438 memb_name = H5Tget_member_name(memtype,u);
439 if(memb_name ==
nullptr)
440 throw InternalErr (__FILE__, __LINE__,
"Fail to obtain the name of an HDF5 compound datatype member.");
442 BaseType *field = h5s->var(memb_name);
443 if (memb_cls == H5T_COMPOUND) {
445 memb_h5s.do_structure_read(dsetid, memb_id,values,has_values,memb_offset+values_offset+ty_size*element);
448 else if(memb_cls == H5T_ARRAY) {
451 int at_ndims = H5Tget_array_ndims(memb_id);
453 throw InternalErr (__FILE__, __LINE__,
"Fail to obtain number of dimensions of the array datatype.");
459 vector<int> at_offset(at_ndims,0);
460 vector<int> at_count(at_ndims,0);
461 vector<int> at_step(at_ndims,0);
463 int at_nelms = h5_array_type.format_constraint(at_offset.data(),at_step.data(),at_count.data());
466 h5_array_type.do_h5_array_type_read(dsetid,memb_id,values,has_values,memb_offset+values_offset+ty_size*element,
467 at_nelms,at_offset.data(),at_count.data(),at_step.data());
470 else if(memb_cls == H5T_INTEGER || memb_cls == H5T_FLOAT) {
472 if(
true == promote_char_to_short(memb_cls,memb_id)) {
473 void *src = (
void*)(values.data() + (element*ty_size) + values_offset +memb_offset);
475 memcpy(&val_int8,src,1);
476 short val_short=(short)val_int8;
477 field->val2buf(&val_short);
480 field->val2buf(values.data() + (element*ty_size) + values_offset +memb_offset);
484 else if(memb_cls == H5T_STRING) {
487 if(
true == H5Tis_variable_str(memb_id)) {
488 void *src = (
void*)(values.data()+(element*ty_size)+values_offset + memb_offset);
490 get_vlen_str_data((
char*)src,final_str);
491 field->val2buf(&final_str);
494 void *src = (
void*)(values.data()+(element*ty_size)+values_offset + memb_offset);
495 vector<char> str_val;
496 size_t memb_size = H5Tget_size(memb_id);
497 if (memb_size == 0) {
499 H5free_memory(memb_name);
501 throw InternalErr (__FILE__, __LINE__,
"Fail to obtain the size of HDF5 compound datatype.");
503 str_val.resize(memb_size);
504 memcpy(str_val.data(),src,memb_size);
505 string temp_string(str_val.begin(),str_val.end());
506 field->val2buf(&temp_string);
510 H5free_memory(memb_name);
513 throw InternalErr (__FILE__, __LINE__,
514 "Only support the field of compound datatype when the field type class is integer, float, string, array or compound..");
520 H5free_memory(memb_name);
521 field->set_read_p(
true);
523 h5s->set_read_p(
true);
524 set_vec(element,h5s);
528 if(
true == has_values) {
530 throw InternalErr(__FILE__, __LINE__,
"memory type and memory space for this compound datatype should be valid.");
532 if(H5Dvlen_reclaim(memtype,mspace,H5P_DEFAULT,(
void*)values.data())<0)
533 throw InternalErr(__FILE__, __LINE__,
"Unable to reclaim the compound datatype array.");
546 if(memb_name !=
nullptr)
547 H5free_memory(memb_name);
550 if(
true == has_values) {
551 if(H5Dvlen_reclaim(memtype,mspace,H5P_DEFAULT,(
void*)(values.data()))<0) {
569bool HDF5Array::m_array_of_reference(hid_t dset_id,hid_t dtype_id)
572#if (H5_VERS_MAJOR == 1 && (H5_VERS_MINOR == 10 || H5_VERS_MINOR == 8 || H5_VERS_MINOR == 6))
573 hid_t d_dset_id = dset_id;
574 hdset_reg_ref_t *rbuf =
nullptr;
578 vector<int> offset(d_num_dim);
579 vector<int> count(d_num_dim);
580 vector<int> step(d_num_dim);
583 int nelms = format_constraint(offset.data(), step.data(), count.data());
584 vector<string> v_str(nelms);
586 BESDEBUG(
"h5",
"=read() URL type is detected. "
587 <<
"nelms=" << nelms <<
" full_size=" << d_num_elm << endl);
591 if (H5Tequal(dtype_id, H5T_STD_REF_DSETREG) < 0) {
592 throw InternalErr(__FILE__, __LINE__,
"H5Tequal() failed");
595 if (H5Tequal(dtype_id, H5T_STD_REF_DSETREG) > 0) {
596 BESDEBUG(
"h5",
"=read() Got regional reference. " << endl);
598 rbuf =
new hdset_reg_ref_t[d_num_elm];
600 throw InternalErr(__FILE__, __LINE__,
"new() failed.");
602 if (H5Dread(d_dset_id, H5T_STD_REF_DSETREG, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) {
603 throw InternalErr(__FILE__, __LINE__,
"H5Dread() failed.");
606 for (
int i = 0; i < nelms; i++) {
608 BESDEBUG(
"h5",
"=read() rbuf[" << i <<
"]" <<
609 rbuf[offset[0] + i * step[0]] << endl);
611 if (rbuf[offset[0] + i * step[0]][0] !=
'\0') {
614 hid_t did_r = H5RDEREFERENCE(d_dset_id, H5R_DATASET_REGION, rbuf[offset[0] + i * step[0]]);
616 throw InternalErr(__FILE__, __LINE__,
"H5RDEREFERENCE() failed.");
620 if (H5Iget_name(did_r, (
char *) r_name,
DODS_NAMELEN) < 0) {
621 throw InternalErr(__FILE__, __LINE__,
"H5Iget_name() failed.");
623 BESDEBUG(
"h5",
"=read() dereferenced name is " << r_name
626 string varname(r_name);
627 hid_t space_id = H5Rget_region(did_r, H5R_DATASET_REGION, rbuf[offset[0] + i * step[0]]);
629 throw InternalErr(__FILE__, __LINE__,
"H5Rget_region() failed.");
633 int ndim = H5Sget_simple_extent_ndims(space_id);
635 throw InternalErr(__FILE__, __LINE__,
"H5Sget_simple_extent_ndims() failed.");
638 BESDEBUG(
"h5",
"=read() dim is " << ndim << endl);
641 switch (H5Sget_select_type(space_id)) {
644 BESDEBUG(
"h5",
"=read() None selected." << endl);
647 case H5S_SEL_POINTS: {
648 BESDEBUG(
"h5",
"=read() Points selected." << endl);
649 hssize_t npoints = H5Sget_select_npoints(space_id);
651 throw InternalErr(__FILE__, __LINE__,
652 "Cannot determine number of elements in the dataspace selection");
655 BESDEBUG(
"h5",
"=read() npoints are " << npoints
657 vector<hsize_t> buf(npoints * ndim);
658 if (H5Sget_select_elem_pointlist(space_id, 0, npoints, buf.data()) < 0) {
659 throw InternalErr(__FILE__, __LINE__,
"H5Sget_select_elem_pointlist() failed.");
663 for (
int j = 0; j < npoints * ndim; j++) {
664 "h5",
"=read() npoints buf[0] =" << buf[j] <<endl;
668 for (
int j = 0; j < (
int) npoints; j++) {
670 expression.append(varname);
671 for (
int k = 0; k < ndim; k++) {
673 oss <<
"[" << (
int) buf[j * ndim + k] <<
"]";
674 expression.append(oss.str());
676 if (j != (
int) (npoints - 1)) {
677 expression.append(
",");
680 v_str[i].append(expression);
684 case H5S_SEL_HYPERSLABS: {
685 vector<hsize_t> start(ndim);
686 vector<hsize_t> end(ndim);
687 vector<hsize_t>stride(ndim);
688 vector<hsize_t>s_count(ndim);
689 vector<hsize_t>block(ndim);
691 BESDEBUG(
"h5",
"=read() Slabs selected." << endl);
692 BESDEBUG(
"h5",
"=read() nblock is " <<
693 H5Sget_select_hyper_nblocks(space_id) << endl);
695#if (H5_VERS_MAJOR == 1 && H5_VERS_MINOR == 8)
696 if (H5Sget_select_bounds(space_id, start.data(), end.data()) < 0) {
697 throw InternalErr(__FILE__, __LINE__,
"H5Sget_select_bounds() failed.");
700 if (H5Sget_regular_hyperslab(space_id, start.data(), stride.data(), s_count.data(), block.data()) < 0) {
701 throw InternalErr(__FILE__, __LINE__,
"H5Sget_regular_hyperslab() failed.");
705 for (
int j = 0; j < ndim; j++) {
707 BESDEBUG(
"h5",
"start " << start[j]
708 <<
"stride "<<stride[j]
709 <<
"count "<< s_count[j]
710 <<
"block "<< block[j]
714 end[j] = start[j] + stride[j]*(s_count[j]-1)+(block[j]-1);
715 BESDEBUG(
"h5",
"=read() start is " << start[j]
716 <<
"=read() end is " << end[j] << endl);
717 oss <<
"[" << start[j] <<
":" << stride[j] <<
":" << end[j] <<
"]";
718 expression.append(oss.str());
719 BESDEBUG(
"h5",
"=read() expression is "
720 << expression << endl)
724 if (!expression.empty()) {
725 v_str[i].append(expression);
731 BESDEBUG(
"h5",
"=read() All selected." << endl);
735 BESDEBUG(
"h5",
"Unknown space type." << endl);
748 if (H5Tequal(dtype_id, H5T_STD_REF_OBJ) < 0) {
749 throw InternalErr(__FILE__, __LINE__,
"H5Tequal() failed.");
752 if (H5Tequal(dtype_id, H5T_STD_REF_OBJ) > 0) {
753 BESDEBUG(
"h5",
"=read() Got object reference. " << endl);
754 vector<hobj_ref_t> orbuf;
755 orbuf.resize(d_num_elm);
756 if (H5Dread(d_dset_id, H5T_STD_REF_OBJ, H5S_ALL, H5S_ALL, H5P_DEFAULT, orbuf.data()) < 0) {
757 throw InternalErr(__FILE__, __LINE__,
"H5Dread failed()");
760 for (
int i = 0; i < nelms; i++) {
762 hid_t did_r = H5RDEREFERENCE(d_dset_id, H5R_OBJECT, &orbuf[offset[0] + i * step[0]]);
764 throw InternalErr(__FILE__, __LINE__,
"H5RDEREFERENCE() failed.");
767 if (H5Iget_name(did_r, (
char *) r_name,
DODS_NAMELEN) < 0) {
768 throw InternalErr(__FILE__, __LINE__,
"H5Iget_name() failed.");
772 string varname(r_name);
774 BESDEBUG(
"h5",
"=read() dereferenced name is " << r_name <<endl);
778 set_value(v_str.data(), nelms);
787 return m_array_of_reference_new_h5_apis(dset_id,dtype_id);
792bool HDF5Array::m_array_of_reference_new_h5_apis(hid_t dset_id,hid_t dtype_id) {
794#if (H5_VERS_MAJOR == 1 && (H5_VERS_MINOR == 10 || H5_VERS_MINOR == 8 || H5_VERS_MINOR == 6))
795 throw InternalErr(__FILE__, __LINE__,
796 "The HDF5 handler compiled with earlier version (<=110)of the HDF5 library should not call method that uses new reference APIs");
800 H5R_ref_t *rbuf =
nullptr;
807 vector<int> offset(d_num_dim);
808 vector<int> count(d_num_dim);
809 vector<int> step(d_num_dim);
810 vector<hsize_t> hoffset(d_num_dim);
811 vector<hsize_t>hcount(d_num_dim);
812 vector<hsize_t>hstep(d_num_dim);
814 int nelms = format_constraint(offset.data(), step.data(), count.data());
815 for (
int i = 0; i <d_num_dim; i++) {
816 hoffset[i] = (hsize_t) offset[i];
817 hcount[i] = (hsize_t) count[i];
818 hstep[i] = (hsize_t) step[i];
821 BESDEBUG(
"h5",
"=read() URL type is detected. "
822 <<
"nelms=" << nelms << endl);
824 rbuf =
new H5R_ref_t[nelms];
826 file_space_id = H5Dget_space(dset_id);
827 if(file_space_id < 0)
828 throw InternalErr(__FILE__, __LINE__,
"Fail to obtain reference dataset file space.");
830 if (H5Sselect_hyperslab(file_space_id, H5S_SELECT_SET,
831 hoffset.data(), hstep.data(),
832 hcount.data(),
nullptr) < 0)
833 throw InternalErr (__FILE__, __LINE__,
"Fail to select the hyperslab for reference dataset.");
836 mem_space_id = H5Screate_simple(d_num_dim,hcount.data(),
nullptr);
838 throw InternalErr(__FILE__, __LINE__,
"Fail to obtain reference dataset memory space.");
840 if(H5Dread(dset_id,H5T_STD_REF,mem_space_id,file_space_id,H5P_DEFAULT,&rbuf[0])<0)
841 throw InternalErr(__FILE__, __LINE__,
"Fail to read hyperslab reference dataset.");
843 H5Sclose(mem_space_id);
844 H5Sclose(file_space_id);
847 vector<string> v_str;
849 H5R_type_t ref_type = H5Rget_type((
const H5R_ref_t *)&rbuf[0]);
852 if(ref_type != H5R_OBJECT2 && ref_type !=H5R_DATASET_REGION2)
853 throw InternalErr(__FILE__, __LINE__,
"Unsupported reference: neither object nor region references");
855 for (
int i = 0; i < nelms; i++) {
857 hid_t obj_id = H5Ropen_object((H5R_ref_t *)&rbuf[i], H5P_DEFAULT, H5P_DEFAULT);
859 throw InternalErr(__FILE__, __LINE__,
"Cannot open the object the reference points to");
861 vector<char> objname;
862 ssize_t objnamelen = -1;
863 if ((objnamelen= H5Iget_name(obj_id,
nullptr,0))<=0) {
865 throw InternalErr(__FILE__, __LINE__,
"Cannot obtain the name length of the object the reference points to");
867 objname.resize(objnamelen+1);
868 if ((objnamelen= H5Iget_name(obj_id,objname.data(),objnamelen+1))<=0) {
870 throw InternalErr(__FILE__, __LINE__,
"Cannot obtain the name length of the object the reference points to");
873 string objname_str = string(objname.begin(),objname.end());
874 string trim_objname = objname_str.substr(0,objnamelen);
877 if(ref_type == H5R_OBJECT2)
878 v_str.push_back(trim_objname);
881 if(H5Rget_obj_type3((H5R_ref_t *)&rbuf[i], H5P_DEFAULT, &obj_type) < 0){
883 throw InternalErr(__FILE__, __LINE__,
"H5Rget_obj_type3() failed.");
885 if(obj_type != H5O_TYPE_DATASET) {
887 throw InternalErr(__FILE__, __LINE__,
"Region reference must point to a dataset.");
889 hid_t region_space_id = H5Ropen_region(&rbuf[i],H5P_DEFAULT,H5P_DEFAULT);
890 if (region_space_id < 0) {
892 throw InternalErr(__FILE__, __LINE__,
"Cannot obtain the space ID the reference points to");
895 int ndim = H5Sget_simple_extent_ndims(region_space_id);
897 H5Sclose(region_space_id);
899 throw InternalErr(__FILE__, __LINE__,
"H5Sget_simple_extent_ndims() failed.");
903 switch (H5Sget_select_type(region_space_id)) {
906 BESDEBUG(
"h5",
"=read() None selected." << endl);
909 case H5S_SEL_POINTS: {
910 BESDEBUG(
"h5",
"=read() Points selected." << endl);
911 hssize_t npoints = H5Sget_select_npoints(region_space_id);
913 H5Sclose(region_space_id);
915 throw InternalErr(__FILE__, __LINE__,
916 "Cannot determine number of elements in the dataspace selection");
919 BESDEBUG(
"h5",
"=read() npoints are " << npoints
921 vector<hsize_t> buf(npoints * ndim);
922 if (H5Sget_select_elem_pointlist(region_space_id, 0, npoints, buf.data()) < 0) {
923 H5Sclose(region_space_id);
925 throw InternalErr(__FILE__, __LINE__,
"H5Sget_select_elem_pointlist() failed.");
929 for (
int j = 0; j < npoints * ndim; j++) {
930 "h5",
"=read() npoints buf[0] =" << buf[j] <<endl;
934 for (
int j = 0; j < (
int) npoints; j++) {
936 expression.append(trim_objname);
937 for (
int k = 0; k < ndim; k++) {
939 oss <<
"[" << (
int) buf[j * ndim + k] <<
"]";
940 expression.append(oss.str());
942 if (j != (
int) (npoints - 1)) {
943 expression.append(
",");
946 v_str.push_back(expression);
950 case H5S_SEL_HYPERSLABS: {
951 vector<hsize_t> start(ndim);
952 vector<hsize_t> end(ndim);
953 vector<hsize_t>stride(ndim);
954 vector<hsize_t>s_count(ndim);
955 vector<hsize_t>block(ndim);
957 BESDEBUG(
"h5",
"=read() Slabs selected." << endl);
958 BESDEBUG(
"h5",
"=read() nblock is " <<
959 H5Sget_select_hyper_nblocks(region_space_id) << endl);
961 if (H5Sget_regular_hyperslab(region_space_id, start.data(), stride.data(), s_count.data(), block.data()) < 0) {
962 H5Sclose(region_space_id);
964 throw InternalErr(__FILE__, __LINE__,
"H5Sget_regular_hyperslab() failed.");
967 expression.append(trim_objname);
968 for (
int j = 0; j < ndim; j++) {
970 BESDEBUG(
"h5",
"start " << start[j]
971 <<
"stride "<<stride[j]
972 <<
"count "<< s_count[j]
973 <<
"block "<< block[j]
977 end[j] = start[j] + stride[j]*(s_count[j]-1)+(block[j]-1);
978 BESDEBUG(
"h5",
"=read() start is " << start[j]
979 <<
"=read() end is " << end[j] << endl);
980 oss <<
"[" << start[j] <<
":" << stride[j] <<
":" << end[j] <<
"]";
981 expression.append(oss.str());
982 BESDEBUG(
"h5",
"=read() expression is "
983 << expression << endl)
986 v_str.push_back(expression);
991 BESDEBUG(
"h5",
"=read() All selected." << endl);
995 BESDEBUG(
"h5",
"Unknown space type." << endl);
998 H5Sclose(region_space_id);
1002 for (
int i = 0; i<nelms; i++)
1003 H5Rdestroy(&rbuf[i]);
1005 H5Sclose(mem_space_id);
1006 H5Sclose(file_space_id);
1007 set_value(v_str.data(), nelms);
1013 H5Sclose(mem_space_id);
1014 H5Sclose(file_space_id);
1023 bool is_new_ref =
false;
1024 bool is_obj_ref =
false;
1025 bool is_reg_ref =
false;
1027 htri_t ref_ret = H5Tequal(dtype_id, H5T_STD_REF);
1031 throw InternalErr(__FILE__, __LINE__,
"H5Tequal() failed to compare H5T_STD_REF");
1032 else if (ref_ret > 0)
1035 if(
false == is_new_ref) {
1037 ref_ret = H5Tequal(dtype_id,H5T_STD_REF_OBJ);
1040 throw InternalErr(__FILE__, __LINE__,
"H5Tequal() failed to compare H5T_STD_REF_OBJ");
1041 else if (ref_ret > 0)
1046 ref_ret = H5Tequal(dtype_id,H5T_STD_REF_DSETREG);
1048 throw InternalErr(__FILE__, __LINE__,
"H5Tequal() failed to compare H5T_STD_REF_DSETREG");
1049 else if (ref_ret > 0)
1057 if(is_obj_ref ==
false && is_reg_ref ==
false)
1058 throw InternalErr(__FILE__, __LINE__,
"datatype must be either object ref. or region ref.");
1062void HDF5Array::m_intern_plain_array_data(
char *convbuf,hid_t memtype)
1065 vector<string> v_str(d_num_elm);
1066 size_t elesize = H5Tget_size(memtype);
1068 throw InternalErr(__FILE__, __LINE__,
"H5Tget_size() failed.");
1070 vector<char> strbuf(elesize + 1);
1071 BESDEBUG(
"h5",
"=read()<check_h5str() element size=" << elesize
1072 <<
" d_num_elm=" << d_num_elm << endl);
1074 for (
int strindex = 0; strindex < d_num_elm; strindex++) {
1075 get_strdata(strindex, convbuf, strbuf.data(), (
int)elesize);
1076 BESDEBUG(
"h5",
"=read()<get_strdata() strbuf=" << strbuf.data() << endl);
1077 v_str[strindex] = strbuf.data();
1080 val2buf((
void *) v_str.data());
1084 val2buf((
void *) convbuf);
1089bool HDF5Array::do_h5_array_type_read(hid_t dsetid, hid_t memb_id,vector<char>&values,
bool has_values,
int values_offset,
1090 int at_nelms,
int* at_offset,
int* at_count,
int* at_step){
1093 if(has_values !=
true)
1094 throw InternalErr (__FILE__, __LINE__,
"Only support the retrieval of HDF5 Array datatype values from the parent compound datatype read.");
1096 hid_t at_base_type = H5Tget_super(memb_id);
1097 if(at_base_type < 0) {
1098 throw InternalErr (__FILE__, __LINE__,
"Fail to obtain the basetype of the array datatype.");
1102 int at_ndims = H5Tget_array_ndims(memb_id);
1104 H5Tclose(at_base_type);
1105 throw InternalErr (__FILE__, __LINE__,
"Fail to obtain number of dimensions of the array datatype.");
1108 vector<hsize_t>at_dims_h(at_ndims,0);
1111 if(H5Tget_array_dims(memb_id,at_dims_h.data())<0) {
1112 H5Tclose(at_base_type);
1113 throw InternalErr (__FILE__, __LINE__,
"Fail to obtain dimensions of the array datatype.");
1115 vector<int>at_dims(at_ndims,0);
1116 for(
int i = 0;i<at_ndims;i++) {
1117 at_dims[i] = (
int)at_dims_h[i];
1119 int at_total_nelms = 1;
1120 for (
int i = 0; i <at_ndims; i++)
1121 at_total_nelms = at_total_nelms*at_dims[i];
1123 H5T_class_t array_cls = H5Tget_class(at_base_type);
1124 if(H5T_NO_CLASS == array_cls) {
1125 H5Tclose(at_base_type);
1126 throw InternalErr (__FILE__, __LINE__,
"Fail to obtain the datatype class of the array base type.");
1129 size_t at_base_type_size = H5Tget_size(at_base_type);
1130 if(0 == at_base_type_size){
1131 H5Tclose(at_base_type);
1132 throw InternalErr (__FILE__, __LINE__,
"Fail to obtain the size of the array base type.");
1136 if(H5T_COMPOUND == array_cls) {
1139 vector<int>at_end(at_ndims,0);
1140 vector<int>at_pos(at_ndims,0);
1141 for (
int i = 0; i< at_ndims; i++){
1142 at_pos[i] = at_offset[i];
1143 at_end[i] = at_offset[i] + (at_count[i] -1)*at_step[i];
1146 int at_orig_index = INDEX_nD_TO_1D(at_dims,at_pos);
1149 for (
int array_index = 0; array_index <at_nelms; array_index++) {
1153 hid_t child_memb_id;
1154 H5T_class_t child_memb_cls;
1156 size_t child_memb_offset;
1158 if((child_nmembs = H5Tget_nmembers(at_base_type)) < 0) {
1159 H5Tclose(at_base_type);
1161 throw InternalErr (__FILE__, __LINE__,
"Fail to obtain number of HDF5 compound datatype.");
1164 for(
unsigned child_u = 0; child_u < (unsigned)child_nmembs; child_u++) {
1167 if((child_memb_id = H5Tget_member_type(at_base_type, child_u)) < 0) {
1168 H5Tclose(at_base_type);
1170 throw InternalErr (__FILE__, __LINE__,
"Fail to obtain the datatype of an HDF5 compound datatype member.");
1174 if((child_memb_cls = H5Tget_member_class (at_base_type, child_u)) < 0) {
1175 H5Tclose(child_memb_id);
1176 H5Tclose(at_base_type);
1178 throw InternalErr (__FILE__, __LINE__,
"Fail to obtain the datatype class of an HDF5 compound datatype member.");
1182 child_memb_offset= H5Tget_member_offset(at_base_type,child_u);
1185 char *child_memb_name = H5Tget_member_name(at_base_type,child_u);
1186 if(child_memb_name ==
nullptr) {
1187 H5Tclose(child_memb_id);
1188 H5Tclose(at_base_type);
1190 throw InternalErr (__FILE__, __LINE__,
"Fail to obtain the name of an HDF5 compound datatype member.");
1193 BaseType *field = h5s->var(child_memb_name);
1194 if (child_memb_cls == H5T_COMPOUND) {
1201 if(at_total_nelms == at_nelms) {
1202 memb_h5s.do_structure_read(dsetid,child_memb_id, values,has_values,values_offset+at_base_type_size*array_index+child_memb_offset);
1209 memb_h5s.do_structure_read(dsetid, child_memb_id, values,has_values,values_offset+at_base_type_size*at_orig_index+child_memb_offset);
1214 else if(child_memb_cls == H5T_ARRAY) {
1217 int child_at_ndims = H5Tget_array_ndims(child_memb_id);
1218 if(child_at_ndims <= 0) {
1219 H5Tclose(at_base_type);
1220 H5Tclose(child_memb_id);
1221 H5free_memory(child_memb_name);
1223 throw InternalErr (__FILE__, __LINE__,
"Fail to obtain number of dimensions of the array datatype.");
1228 vector<int> child_at_offset(child_at_ndims,0);
1229 vector<int> child_at_count(child_at_ndims,0);
1230 vector<int> child_at_step(child_at_ndims,0);
1232 int child_at_nelms = h5_array_type.format_constraint(child_at_offset.data(),child_at_step.data(),child_at_count.data());
1234 if(at_total_nelms == at_nelms) {
1235 h5_array_type.do_h5_array_type_read(dsetid,child_memb_id,values,has_values,child_memb_offset+values_offset+at_base_type_size*array_index,
1236 child_at_nelms,child_at_offset.data(),child_at_count.data(),child_at_step.data());
1239 h5_array_type.do_h5_array_type_read(dsetid,child_memb_id,values,has_values,child_memb_offset+values_offset+at_base_type_size*at_orig_index,
1240 child_at_nelms,child_at_offset.data(),child_at_count.data(),child_at_step.data());
1245 else if(H5T_INTEGER == child_memb_cls || H5T_FLOAT == child_memb_cls){
1247 int number_index =((at_total_nelms == at_nelms)?array_index:at_orig_index);
1248 if(
true == promote_char_to_short(child_memb_cls,child_memb_id)) {
1249 void *src = (
void*)(values.data() + (number_index*at_base_type_size) + values_offset +child_memb_offset);
1251 memcpy(&val_int8,src,1);
1252 short val_short=(short)val_int8;
1253 field->val2buf(&val_short);
1256 field->val2buf(values.data() + (number_index * at_base_type_size) + values_offset+child_memb_offset);
1260 else if(H5T_STRING == child_memb_cls){
1262 int string_index =((at_total_nelms == at_nelms)?array_index:at_orig_index);
1265 if(
true == H5Tis_variable_str(child_memb_id)) {
1268 void *src = (
void*)(values.data()+(string_index *at_base_type_size)+values_offset+child_memb_offset);
1270 char*temp_bp =(
char*)src;
1271 get_vlen_str_data(temp_bp,final_str);
1272 field->val2buf(&final_str[0]);
1274 field->set_value(final_str);
1278 void *src = (
void*)(values.data()+(string_index *at_base_type_size)+values_offset+child_memb_offset);
1279 vector<char> str_val;
1280 size_t memb_size = H5Tget_size(child_memb_id);
1281 if (memb_size == 0) {
1282 H5Tclose(child_memb_id);
1283 H5Tclose(at_base_type);
1284 H5free_memory(child_memb_name);
1286 throw InternalErr (__FILE__, __LINE__,
"Fail to obtain the size of HDF5 compound datatype.");
1288 str_val.resize(memb_size);
1289 memcpy(str_val.data(),src,memb_size);
1290 field->val2buf(str_val.data());
1295 H5Tclose(child_memb_id);
1296 H5Tclose(at_base_type);
1297 H5free_memory(child_memb_name);
1299 throw InternalErr (__FILE__, __LINE__,
"Unsupported datatype class for the array base type.");
1303 field->set_read_p(
true);
1304 H5free_memory(child_memb_name);
1305 H5Tclose(child_memb_id);
1308 h5s->set_read_p(
true);
1311 set_vec(array_index,h5s);
1314 vector<int>at_offsetv(at_pos.size(),0);
1315 vector<int>at_stepv(at_pos.size(),0);
1316 for (
unsigned int at_index = 0; at_index<at_pos.size();at_index++){
1317 at_offsetv[at_index] = at_offset[at_index];
1318 at_stepv[at_index] = at_step[at_index];
1321 obtain_next_pos(at_pos,at_offsetv,at_end,at_stepv,(
int)(at_pos.size()));
1322 at_orig_index = INDEX_nD_TO_1D(at_dims,at_pos);
1330 else if(H5T_INTEGER == array_cls|| H5T_FLOAT == array_cls) {
1333 if(at_total_nelms == at_nelms) {
1336 if(
true == promote_char_to_short(array_cls ,at_base_type)) {
1337 vector<char> val_int8;
1338 val_int8.resize(at_nelms);
1339 void*src = (
void*)(values.data() +values_offset);
1340 memcpy(val_int8.data(),src,at_nelms);
1342 vector<short> val_short;
1343 for (
int i = 0; i<at_nelms; i++)
1344 val_short[i] = (
short)val_int8[i];
1346 val2buf(val_short.data());
1350 val2buf(values.data() + values_offset);
1356 string dap_type =
get_dap_type(at_base_type,is_dap4());
1359 void*src = (
void*)(values.data() + values_offset);
1362 vector<int>at_pos(at_ndims,0);
1363 for (
int i = 0; i< at_ndims; i++)
1364 at_pos[i] = at_offset[i];
1366 if( BYTE == dap_type) {
1368 vector<unsigned char>total_val;
1369 total_val.resize(at_total_nelms);
1370 memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1372 vector<unsigned char>final_val;
1373 subset<unsigned char>(
1385 set_value(final_val.data(),at_nelms);
1389 else if( INT16 == dap_type) {
1392 if(
true == promote_char_to_short(array_cls,at_base_type)) {
1393 vector<char>total_val;
1394 total_val.resize(at_total_nelms);
1395 memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1397 vector<char>final_val;
1410 vector<short> final_val_short;
1411 final_val_short.resize(at_nelms);
1412 for(
int i = 0; i<at_nelms; i++)
1413 final_val_short[i] = final_val[i];
1415 val2buf(final_val_short.data());
1420 vector<short>total_val;
1421 total_val.resize(at_total_nelms);
1422 memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1424 vector<short>final_val;
1437 val2buf(final_val.data());
1441 else if( UINT16 == dap_type) {
1442 vector<unsigned short>total_val;
1443 total_val.resize(at_total_nelms);
1444 memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1446 vector<unsigned short>final_val;
1447 subset<unsigned short>(
1459 val2buf(final_val.data());
1462 else if(UINT32 == dap_type) {
1463 vector<unsigned int>total_val;
1464 total_val.resize(at_total_nelms);
1465 memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1467 vector<unsigned int>final_val;
1468 subset<unsigned int>(
1479 val2buf(final_val.data());
1483 else if(INT32 == dap_type) {
1484 vector<int>total_val;
1485 total_val.resize(at_total_nelms);
1486 memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1488 vector<int>final_val;
1501 val2buf(final_val.data());
1504 else if(FLOAT32 == dap_type) {
1505 vector<float>total_val;
1506 total_val.resize(at_total_nelms);
1507 memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1509 vector<float>final_val;
1522 val2buf(final_val.data());
1525 else if(FLOAT64 == dap_type) {
1526 vector<double>total_val;
1527 total_val.resize(at_total_nelms);
1528 memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1530 vector<double>final_val;
1545 val2buf(final_val.data());
1549 H5Tclose(at_base_type);
1550 throw InternalErr (__FILE__, __LINE__,
1551 "Non-supported integer or float datatypes");
1556 else if(H5T_STRING == array_cls) {
1559 vector<int>at_pos(at_ndims,0);
1560 for (
int i = 0; i< at_ndims; i++)
1561 at_pos[i] = at_offset[i];
1563 vector<string>total_strval;
1564 total_strval.resize(at_total_nelms);
1566 if(
true == H5Tis_variable_str(at_base_type)) {
1567 void *src = (
void*)(values.data()+values_offset);
1568 char*temp_bp =(
char*)src;
1569 for(
int i = 0;i <at_total_nelms; i++){
1571 get_vlen_str_data(temp_bp,tempstrval);
1572 total_strval[i] = tempstrval;
1573 temp_bp += at_base_type_size;
1575 if(at_total_nelms == at_nelms) {
1579 set_value(total_strval,at_total_nelms);
1585 vector<string>final_val;
1587 total_strval.data(),
1598 set_value(final_val,at_nelms);
1604 void *src = (
void*)(values.data()+values_offset);
1605 for(
int i = 0; i <at_total_nelms; i++)
1606 total_strval[i].resize(at_base_type_size);
1608 vector<char> str_val;
1609 str_val.resize(at_total_nelms*at_base_type_size);
1610 memcpy((
void*)str_val.data(),src,at_total_nelms*at_base_type_size);
1611 string total_in_one_string(str_val.begin(),str_val.end());
1612 for(
int i = 0; i<at_total_nelms;i++)
1613 total_strval[i] = total_in_one_string.substr(i*at_base_type_size,at_base_type_size);
1615 if(at_total_nelms == at_nelms)
1616 set_value(total_strval,at_total_nelms);
1618 vector<string>final_val;
1620 total_strval.data(),
1630 set_value(final_val,at_nelms);
1637 H5Tclose(at_base_type);
1638 throw InternalErr (__FILE__, __LINE__,
1639 "Only support the field of compound datatype when the field type class is integer, float, string, array or compound..");
1643 H5Tclose(at_base_type);
1650HDF5Array::INDEX_nD_TO_1D (
const std::vector < int > &dims,
1651 const std::vector < int > &pos)
1657 assert (dims.size () == pos.size ());
1661 for (
unsigned int p = 0; p < pos.size (); p++) {
1664 for (
unsigned int j = start; j < dims.size (); j++)
1673bool HDF5Array::obtain_next_pos(vector<int>& pos, vector<int>&start,vector<int>&end,vector<int>&step,
int rank_change) {
1675 if((pos[rank_change-1] + step[rank_change-1])<=end[rank_change-1]) {
1676 pos[rank_change-1] = pos[rank_change-1] + step[rank_change-1];
1680 if( 1 == rank_change)
1682 pos[rank_change-1] = start[rank_change-1];
1683 obtain_next_pos(pos,start,end,step,rank_change-1);
1700int HDF5Array::subset(
1707 std::vector<T> *poutput,
1711 for(
int k=0; k<edge[index]; k++)
1713 pos[index] = start[index] + k*stride[index];
1715 subset(input, rank, dim, start, stride, edge, poutput,pos,index+1);
1718 poutput->push_back(input[INDEX_nD_TO_1D( dim, pos)]);
1742hid_t HDF5Array::mkstr(
int size, H5T_str_t pad)
1747 if ((str_type = H5Tcopy(H5T_C_S1)) < 0)
1749 if (H5Tset_size(str_type, (
size_t) size) < 0)
1751 if (H5Tset_strpad(str_type, pad) < 0)
1759BaseType* HDF5Array::h5dims_transform_to_dap4(D4Group *grp,
const vector<string> &dimpath) {
1761 BESDEBUG(
"h5",
"<h5dims_transform_to_dap4" << endl);
1773 for (Array::Dim_iter d = dest->dim_begin(), e = dest->dim_end(); d != e; ++d) {
1775 if (
false == (*d).name.empty()) {
1776 BESDEBUG(
"h5",
"<coming to the dimension loop, has name " << (*d).name<<endl);
1777 BESDEBUG(
"h5",
"<coming to the dimension loop, has dimpath " << dimpath[k] <<endl);
1778 BESDEBUG(
"h5",
"<coming to the dimension loop, has dimpath group " << dimpath[k].substr(0,dimpath[k].find_last_of(
"/")+1) <<endl);
1780 D4Group *temp_grp = grp;
1781 D4Dimension *d4_dim =
nullptr;
1782 bool is_dim_nonc4_grp =
false;
1786 BESDEBUG(
"h5",
"<coming to the group has name " << temp_grp->name()<<endl);
1787 BESDEBUG(
"h5",
"<coming to the group has fullpath " << temp_grp->FQN()<<endl);
1790 D4Dimensions *temp_dims = temp_grp->dims();
1793 d4_dim = temp_dims->find_dim((*d).name);
1796 string d4_dim_path = dimpath[k].substr(0,dimpath[k].find_last_of(
"/")+1);
1797 BESDEBUG(
"h5",
"d4_dim_path is " << d4_dim_path<<endl);
1799 bool ancestor_grp =
false;
1802 if(d4_dim_path.find(temp_grp->FQN())==0 || temp_grp->FQN().find(d4_dim_path)==0)
1803 ancestor_grp =
true;
1807 if(d4_dim && (temp_grp->FQN() == d4_dim_path)) {
1808 BESDEBUG(
"h5",
"<FInd dimension name " << (*d).name<<endl);
1810 is_dim_nonc4_grp =
false;
1818 else if( ancestor_grp ==
false) {
1819 is_dim_nonc4_grp =
true;
1825 if(temp_grp->get_parent())
1826 temp_grp =
static_cast<D4Group*
>(temp_grp->get_parent());
1835 if(
true == is_dim_nonc4_grp) {
1836 string err=
"The variable " + var_path +
" has dimension ";
1837 err += dimpath[k] +
". This dimension is not under its ancestor or the current group.";
1838 err +=
" This is not supported.";
1840 throw InternalErr(__FILE__,__LINE__,err);
1843 bool d4_dim_null = ((d4_dim==
nullptr)?
true:
false);
1844 if(d4_dim_null ==
true) {
1845 d4_dim =
new D4Dimension((*d).name, (*d).size);
1846 D4Dimensions * dims = grp->dims();
1847 BESDEBUG(
"h5",
"<Just before adding D4 dimension to group" << endl);
1848 dims->add_dim_nocopy(d4_dim);
1855 dest->set_is_dap4(
true);
A class for handling all types of array in HDF5 for the default option.
This class that translates HDF5 string into DAP string for the default option.
This class converts HDF5 compound type into DAP structure for the default option.
libdap::BaseType * ptr_duplicate() override
bool read() override
Reads HDF5 array data into local buffer.
void set_numdim(int ndims)
remembers number of dimensions of this array.
HDF5Array(const std::string &n, const std::string &d, libdap::BaseType *v)
Constructor.
void set_numelm(int nelms)
remembers number of elements in this array.
void set_memneed(size_t need)
remembers memory size needed.
void get_data(hid_t dset, void *buf)
void get_strdata(int strindex, char *allbuf, char *buf, int elesize)
bool check_h5str(hid_t h5type)
string get_dap_type(hid_t type, bool is_dap4)
const int DODS_NAMELEN
Maximum length of variable or attribute name(default option only).