bes Updated for version 3.20.13
h5dmr.cc
Go to the documentation of this file.
1// This file is part of hdf5_handler a HDF5 file handler for the OPeNDAP
2// data server.
3
4// Copyright (c) 2007-2015 The HDF Group, Inc. and OPeNDAP, Inc.
5//
6// This is free software; you can redistribute it and/or modify it under the
7// terms of the GNU Lesser General Public License as published by the Free
8// Software Foundation; either version 2.1 of the License, or (at your
9// option) any later version.
10//
11// This software is distributed in the hope that it will be useful, but
12// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14// License for more details.
15//
16// You should have received a copy of the GNU Lesser General Public
17// License along with this library; if not, write to the Free Software
18// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19//
20// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
21// You can contact The HDF Group, Inc. at 1800 South Oak Street,
22// Suite 203, Champaign, IL 61820
23
36// the correct DAP4 DMR layout(group's variables first and then the group).
39
40#include <sstream>
41#include "config_hdf5.h"
42
43#include <libdap/InternalErr.h>
44#include <BESDebug.h>
45
46#include <libdap/mime_util.h>
47
48#include "hdf5_handler.h"
49#include "HDF5Int32.h"
50#include "HDF5UInt32.h"
51#include "HDF5UInt16.h"
52#include "HDF5Int16.h"
53#include "HDF5Byte.h"
54#include "HDF5Array.h"
55#include "HDF5Str.h"
56#include "HDF5Float32.h"
57#include "HDF5Float64.h"
58#include "HDF5Url.h"
59#include "HDF5Structure.h"
60
61// The HDF5CFUtil.h includes the utility function obtain_string_after_lastslash.
62#include "HDF5CFUtil.h"
63#include "h5dmr.h"
64
65using namespace std;
66using namespace libdap;
69
70
72static DS_t dt_inst;
73
75void map_h5_attrs_to_dap4(hid_t oid,D4Group* d4g,BaseType* d4b,Structure * d4s,int flag);
76
77#if 0
83// \param par_grp DAP4 parent group
93
94//bool depth_first(hid_t pid, char *gname, DMR & dmr, D4Group* par_grp, const char *fname)
95bool depth_first(hid_t pid, char *gname, D4Group* par_grp, const char *fname)
96{
97 BESDEBUG("h5",
98 ">depth_first() for dmr "
99 << " pid: " << pid
100 << " gname: " << gname
101 << " fname: " << fname
102 << endl);
103
105 int slinkindex = 0;
106
107 H5G_info_t g_info;
108 hsize_t nelems = 0;
109
111 if(H5Gget_info(pid,&g_info) <0) {
112 string msg =
113 "h5_dmr handler: counting hdf5 group elements error for ";
114 msg += gname;
115 throw InternalErr(__FILE__, __LINE__, msg);
116 }
117
118 nelems = g_info.nlinks;
119
120 ssize_t oname_size = 0;
121
122 // Iterate through the file to see the members of the group from the root.
123 for (hsize_t i = 0; i < nelems; i++) {
124
125 vector <char>oname;
126
127 // Query the length of object name.
128 oname_size =
129 H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,nullptr,
130 (size_t)DODS_NAMELEN, H5P_DEFAULT);
131 if (oname_size <= 0) {
132 string msg = "h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
133 msg += gname;
134 throw InternalErr(__FILE__, __LINE__, msg);
135 }
136
137 // Obtain the name of the object
138 oname.resize((size_t) oname_size + 1);
139
140 if (H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,oname.data(),
141 (size_t)(oname_size+1), H5P_DEFAULT) < 0){
142 string msg =
143 "h5_dmr handler: Error getting the hdf5 object name from the group: ";
144 msg += gname;
145 throw InternalErr(__FILE__, __LINE__, msg);
146 }
147
148 // Check if it is the hard link or the soft link
149 H5L_info_t linfo;
150 if (H5Lget_info(pid,oname.data(),&linfo,H5P_DEFAULT)<0) {
151 string msg = "hdf5 link name error from: ";
152 msg += gname;
153 throw InternalErr(__FILE__, __LINE__, msg);
154 }
155
156 // Information of soft links are stored as attributes
157 if(linfo.type == H5L_TYPE_SOFT) {
158 slinkindex++;
159 size_t val_size = linfo.u.val_size;
160 get_softlink(par_grp,pid,oname.data(),slinkindex,val_size);
161 //get_softlink(par_grp,pid,gname,oname.data(),slinkindex,val_size);
162 continue;
163 }
164
165 // Ignore external links
166 if(linfo.type == H5L_TYPE_EXTERNAL)
167 continue;
168
169 // Obtain the object type, such as group or dataset.
170 H5O_info_t oinfo;
171
172 if (H5OGET_INFO_BY_IDX(pid, ".", H5_INDEX_NAME, H5_ITER_NATIVE,
173 i, &oinfo, H5P_DEFAULT)<0) {
174 string msg = "h5_dmr handler: Error obtaining the info for the object";
175 msg += string(oname.begin(),oname.end());
176 throw InternalErr(__FILE__, __LINE__, msg);
177 }
178
179 H5O_type_t obj_type = oinfo.type;
180
181 switch (obj_type) {
182
183 case H5O_TYPE_GROUP:
184 {
185
186 // Obtain the full path name
187 string full_path_name =
188 string(gname) + string(oname.begin(),oname.end()-1) + "/";
189
190 BESDEBUG("h5", "=depth_first dmr ():H5G_GROUP " << full_path_name
191 << endl);
192
193 vector <char>t_fpn;
194 t_fpn.resize(full_path_name.length()+1);
195 copy(full_path_name.begin(),full_path_name.end(),t_fpn.begin());
196 t_fpn[full_path_name.length()] = '\0';
197
198 hid_t cgroup = H5Gopen(pid, t_fpn.data(),H5P_DEFAULT);
199 if (cgroup < 0){
200 throw InternalErr(__FILE__, __LINE__, "h5_dmr handler: H5Gopen() failed.");
201 }
202
203 string grp_name = string(oname.begin(),oname.end()-1);
204
205 // Check the hard link loop and break the loop if it exists.
206 string oid = get_hardlink_dmr(cgroup, full_path_name.c_str());
207 if (oid == "") {
208 try {
209 D4Group* tem_d4_cgroup = new D4Group(grp_name);
210 // Map the HDF5 cgroup attributes to DAP4 group attributes.
211 // Note the last flag of map_h5_attrs_to_dap4 must be 0 for the group attribute mapping.
212 map_h5_attrs_to_dap4(cgroup,tem_d4_cgroup,nullptr,nullptr,0);
213
214 // Add this new DAP4 group
215 par_grp->add_group_nocopy(tem_d4_cgroup);
216
217 // Continue searching the objects under this group
218 //depth_first(cgroup, t_fpn.data(), dmr, tem_d4_cgroup,fname);
219 depth_first(cgroup, t_fpn.data(), tem_d4_cgroup,fname);
220 }
221 catch(...) {
222 H5Gclose(cgroup);
223 throw;
224 }
225 }
226 else {
227 // This group has been visited.
228 // Add the attribute table with the attribute name as HDF5_HARDLINK.
229 // The attribute value is the name of the group when it is first visited.
230 D4Group* tem_d4_cgroup = new D4Group(string(grp_name));
231
232 // Note attr_str_c is the DAP4 attribute string datatype
233 D4Attribute *d4_hlinfo = new D4Attribute("HDF5_HARDLINK",attr_str_c);
234
235 d4_hlinfo->add_value(obj_paths.get_name(oid));
236 tem_d4_cgroup->attributes()->add_attribute_nocopy(d4_hlinfo);
237 par_grp->add_group_nocopy(tem_d4_cgroup);
238
239 }
240
241 if (H5Gclose(cgroup) < 0){
242 throw InternalErr(__FILE__, __LINE__, "Could not close the group.");
243 }
244 break;
245 }
246
247 case H5O_TYPE_DATASET:
248 {
249
250 // Obtain the absolute path of the HDF5 dataset
251 string full_path_name = string(gname) + string(oname.begin(),oname.end()-1);
252
253 // TOOOODOOOO
254 // Obtain the hdf5 dataset handle stored in the structure dt_inst.
255 // All the metadata information in the handler is stored in dt_inst.
256 // Work on this later, redundant for dmr since dataset is opened twice. KY 2015-07-01
257 // Note: depth_first is for building DMR of an HDF5 file that doesn't use dim. scale.
258 // so passing the last parameter as false.
259 get_dataset(pid, full_path_name, &dt_inst,false);
260
261 // Here we open the HDF5 dataset again to use the dataset id for dataset attributes.
262 // This is not necessary for DAP2 since DAS and DDS are separated.
263 hid_t dset_id = -1;
264 if((dset_id = H5Dopen(pid,full_path_name.c_str(),H5P_DEFAULT)) <0) {
265 string msg = "cannot open the HDF5 dataset ";
266 msg += full_path_name;
267 throw InternalErr(__FILE__, __LINE__, msg);
268 }
269
270 try {
271 read_objects(par_grp, full_path_name, fname,dset_id);
272 }
273 catch(...) {
274 H5Dclose(dset_id);
275 throw;
276 }
277 if(H5Dclose(dset_id)<0) {
278 string msg = "cannot close the HDF5 dataset ";
279 msg += full_path_name;
280 throw InternalErr(__FILE__, __LINE__, msg);
281 }
282 }
283 break;
284
285 case H5O_TYPE_NAMED_DATATYPE:
286 // ignore the named datatype
287 break;
288 default:
289 break;
290 }// switch(obj_type)
291 } // for i is 0 ... nelems
292
293 BESDEBUG("h5", "<depth_first() for dmr" << endl);
294 return true;
295}
296#endif
303// \param par_grp DAP4 parent group
316
317
318// The reason to use breadth_first is that the DMR representation needs to show the dimension names and the variables under the group first and then the group names.
319// So we use this search. In the future, we may just use the breadth_first search for all cases.??
320//bool breadth_first(hid_t pid, char *gname, DMR & dmr, D4Group* par_grp, const char *fname,bool use_dimscale)
321bool breadth_first(const hid_t file_id, hid_t pid, const char *gname, D4Group* par_grp, const char *fname,bool use_dimscale,vector<link_info_t> & hdf5_hls )
322{
323 BESDEBUG("h5",
324 ">breadth_first() for dmr "
325 << " pid: " << pid
326 << " gname: " << gname
327 << " fname: " << fname
328 << endl);
329
331 int slinkindex = 0;
332
333 // Obtain the number of objects in this group
334 H5G_info_t g_info;
335 hsize_t nelems = 0;
336 if(H5Gget_info(pid,&g_info) <0) {
337 string msg =
338 "h5_dmr handler: counting hdf5 group elements error for ";
339 msg += gname;
340 throw InternalErr(__FILE__, __LINE__, msg);
341 }
342
343 nelems = g_info.nlinks;
344
345 ssize_t oname_size;
346
347 // First iterate through the HDF5 datasets under the group.
348 for (hsize_t i = 0; i < nelems; i++) {
349
350 vector <char>oname;
351
352 // Query the length of object name.
353 oname_size =
354 H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,nullptr,
355 (size_t)DODS_NAMELEN, H5P_DEFAULT);
356 if (oname_size <= 0) {
357 string msg = "h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
358 msg += gname;
359 throw InternalErr(__FILE__, __LINE__, msg);
360 }
361
362 // Obtain the name of the object
363 oname.resize((size_t) oname_size + 1);
364
365 if (H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,oname.data(),
366 (size_t)(oname_size+1), H5P_DEFAULT) < 0){
367 string msg =
368 "h5_dmr handler: Error getting the hdf5 object name from the group: ";
369 msg += gname;
370 throw InternalErr(__FILE__, __LINE__, msg);
371 }
372
373 // Check if it is the hard link or the soft link
374 H5L_info_t linfo;
375 if (H5Lget_info(pid,oname.data(),&linfo,H5P_DEFAULT)<0) {
376 string msg = "hdf5 link name error from: ";
377 msg += gname;
378 throw InternalErr(__FILE__, __LINE__, msg);
379 }
380
381 // Information of soft links are stored as attributes
382 if(linfo.type == H5L_TYPE_SOFT) {
383 slinkindex++;
384
385 // Size of a soft link value
386 size_t val_size = linfo.u.val_size;
387 get_softlink(par_grp,pid,oname.data(),slinkindex,val_size);
388 continue;
389 }
390
391 // Ignore external links
392 if(linfo.type == H5L_TYPE_EXTERNAL)
393 continue;
394
395 // Obtain the object type, such as group or dataset.
396 H5O_info_t oinfo;
397
398 if (H5OGET_INFO_BY_IDX(pid, ".", H5_INDEX_NAME, H5_ITER_NATIVE,
399 i, &oinfo, H5P_DEFAULT)<0) {
400 string msg = "h5_dmr handler: Error obtaining the info for the object";
401 msg += string(oname.begin(),oname.end());
402 throw InternalErr(__FILE__, __LINE__, msg);
403 }
404
405 H5O_type_t obj_type = oinfo.type;
406
407 if(H5O_TYPE_DATASET == obj_type) {
408
409 // Obtain the absolute path of the HDF5 dataset
410 string full_path_name = string(gname) + string(oname.begin(),oname.end()-1);
411
412 // TOOOODOOOO
413 // Obtain the hdf5 dataset handle stored in the structure dt_inst.
414 // All the metadata information in the handler is stored in dt_inst.
415 // Work on this later, redundant for dmr since dataset is opened twice. KY 2015-07-01
416 // Dimension scale is handled in this routine. So need to keep it. KY 2020-06-10
417 bool is_pure_dim = false;
418 get_dataset_dmr(file_id, pid, full_path_name, &dt_inst,use_dimscale,is_pure_dim,hdf5_hls);
419
420 if(false == is_pure_dim) {
421 hid_t dset_id = -1;
422 if((dset_id = H5Dopen(pid,full_path_name.c_str(),H5P_DEFAULT)) <0) {
423 string msg = "cannot open the HDF5 dataset ";
424 msg += full_path_name;
425 throw InternalErr(__FILE__, __LINE__, msg);
426 }
427
428 try {
429 read_objects(par_grp, full_path_name, fname,dset_id);
430 }
431 catch(...) {
432 H5Dclose(dset_id);
433 throw;
434 }
435 if(H5Dclose(dset_id)<0) {
436 string msg = "cannot close the HDF5 dataset ";
437 msg += full_path_name;
438 throw InternalErr(__FILE__, __LINE__, msg);
439 }
440 }
441 else {
442 //Need to add this pure dimension to the corresponding DAP4 group
443 D4Dimensions *d4_dims = par_grp->dims();
444 auto d4dim_name = string(oname.begin(),oname.end()-1);
445 D4Dimension *d4_dim = d4_dims->find_dim(d4dim_name);
446 if(d4_dim == nullptr) {
447 d4_dim = new D4Dimension(d4dim_name,dt_inst.nelmts);
448 d4_dims->add_dim_nocopy(d4_dim);
449 }
450 BESDEBUG("h5", "<h5dmr.cc: pure dimension: dataset name." << d4dim_name << endl);
451 if(H5Tclose(dt_inst.type)<0) {
452 throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 datatype.");
453 }
454 }
455
456 }
457 }
458
459 // The attributes of this group. Doing this order to follow ncdump's way (variable,attribute then groups)
460 map_h5_attrs_to_dap4(pid,par_grp,nullptr,nullptr,0);
461
462 // Then HDF5 child groups
463 for (hsize_t i = 0; i < nelems; i++) {
464
465 vector <char>oname;
466
467 // Query the length of object name.
468 oname_size =
469 H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,nullptr,
470 (size_t)DODS_NAMELEN, H5P_DEFAULT);
471 if (oname_size <= 0) {
472 string msg = "h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
473 msg += gname;
474 throw InternalErr(__FILE__, __LINE__, msg);
475 }
476
477 // Obtain the name of the object
478 oname.resize((size_t) oname_size + 1);
479
480 if (H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,oname.data(),
481 (size_t)(oname_size+1), H5P_DEFAULT) < 0){
482 string msg =
483 "h5_dmr handler: Error getting the hdf5 object name from the group: ";
484 msg += gname;
485 throw InternalErr(__FILE__, __LINE__, msg);
486 }
487
488 // Check if it is the hard link or the soft link
489 H5L_info_t linfo;
490 if (H5Lget_info(pid,oname.data(),&linfo,H5P_DEFAULT)<0) {
491 string msg = "hdf5 link name error from: ";
492 msg += gname;
493 throw InternalErr(__FILE__, __LINE__, msg);
494 }
495
496 // Information of soft links are handled already, the softlinks need to be ignored, otherwise
497 // the group it links will be mapped again in the block of if obj_type is H5O_TYPE_GROUP
498 if(linfo.type == H5L_TYPE_SOFT) {
499 continue;
500 }
501
502 // Ignore external links
503 if(linfo.type == H5L_TYPE_EXTERNAL)
504 continue;
505
506 // Obtain the object type, such as group or dataset.
507 H5O_info_t oinfo;
508
509 if (H5OGET_INFO_BY_IDX(pid, ".", H5_INDEX_NAME, H5_ITER_NATIVE,
510 i, &oinfo, H5P_DEFAULT)<0) {
511 string msg = "h5_dmr handler: Error obtaining the info for the object in the breadth_first.";
512 throw InternalErr(__FILE__, __LINE__, msg);
513 }
514
515 H5O_type_t obj_type = oinfo.type;
516
517
518 if(obj_type == H5O_TYPE_GROUP) {
519
520 // Obtain the full path name
521 string full_path_name =
522 string(gname) + string(oname.begin(),oname.end()-1) + "/";
523
524 BESDEBUG("h5", "=breadth_first dmr ():H5G_GROUP " << full_path_name
525 << endl);
526
527 vector <char>t_fpn;
528 t_fpn.resize(full_path_name.length()+1);
529 copy(full_path_name.begin(),full_path_name.end(),t_fpn.begin());
530 t_fpn[full_path_name.length()] = '\0';
531
532 hid_t cgroup = H5Gopen(pid, t_fpn.data(),H5P_DEFAULT);
533 if (cgroup < 0){
534 throw InternalErr(__FILE__, __LINE__, "h5_dmr handler: H5Gopen() failed.");
535 }
536
537 auto grp_name = string(oname.begin(),oname.end()-1);
538
539 // Check the hard link loop and break the loop if it exists.
540 string oid = get_hardlink_dmr(cgroup, full_path_name.c_str());
541 if (oid == "") {
542 try {
543 auto tem_d4_cgroup = new D4Group(grp_name);
544
545 // Add this new DAP4 group
546 par_grp->add_group_nocopy(tem_d4_cgroup);
547
548 // Continue searching the objects under this group
549 breadth_first(file_id,cgroup, t_fpn.data(), tem_d4_cgroup,fname,use_dimscale,hdf5_hls);
550 }
551 catch(...) {
552 H5Gclose(cgroup);
553 throw;
554 }
555 }
556 else {
557 // This group has been visited.
558 // Add the attribute table with the attribute name as HDF5_HARDLINK.
559 // The attribute value is the name of the group when it is first visited.
560 D4Group* tem_d4_cgroup = new D4Group(string(grp_name));
561
562 // Note attr_str_c is the DAP4 attribute string datatype
563 D4Attribute *d4_hlinfo = new D4Attribute("HDF5_HARDLINK",attr_str_c);
564
565 d4_hlinfo->add_value(obj_paths.get_name(oid));
566 tem_d4_cgroup->attributes()->add_attribute_nocopy(d4_hlinfo);
567 par_grp->add_group_nocopy(tem_d4_cgroup);
568 }
569
570 if (H5Gclose(cgroup) < 0){
571 throw InternalErr(__FILE__, __LINE__, "Could not close the group.");
572 }
573 }// end if
574 } // for i is 0 ... nelems
575
576 BESDEBUG("h5", "<breadth_first() " << endl);
577 return true;
578}
579
595//
596void
597read_objects( D4Group * d4_grp, const string &varname, const string &filename, const hid_t dset_id)
598{
599
600 switch (H5Tget_class(dt_inst.type)) {
601
602 // HDF5 compound maps to DAP structure.
603 case H5T_COMPOUND:
604 read_objects_structure(d4_grp, varname, filename,dset_id);
605 break;
606
607 case H5T_ARRAY:
608 H5Tclose(dt_inst.type);
609 throw InternalErr(__FILE__, __LINE__, "Currently don't support accessing data of Array datatype when array datatype is not inside the compound.");
610
611 default:
612 read_objects_base_type(d4_grp,varname, filename,dset_id);
613 break;
614 }
615 // We must close the datatype obtained in the get_dataset routine since this is the end of reading DDS.
616 if(H5Tclose(dt_inst.type)<0) {
617 throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 datatype.");
618 }
619}
620
635//
636
637//void
638//read_objects_base_type(DMR & dmr, D4Group * d4_grp,const string & varname,
639void
640read_objects_base_type(D4Group * d4_grp,const string & varname,
641 const string & filename,hid_t dset_id)
642{
643
644 // Obtain the relative path of the variable name under the leaf group
645 string newvarname = HDF5CFUtil::obtain_string_after_lastslash(varname);
646
647 // Get a base type. It should be an HDF5 atomic datatype
648 // datatype.
649 BaseType *bt = Get_bt(newvarname, varname,filename, dt_inst.type,true);
650 if (!bt) {
651 throw
652 InternalErr(__FILE__, __LINE__,
653 "Unable to convert hdf5 datatype to dods basetype");
654 }
655
656 // First deal with scalar data.
657 if (dt_inst.ndims == 0) {
658 // transform the DAP2 to DAP4 for this DAP base type and add it to d4_grp
659 bt->transform_to_dap4(d4_grp,d4_grp);
660 // Get it back - this may return null because the underlying type
661 // may have no DAP2 manifestation.
662 BaseType* new_var = d4_grp->var(bt->name());
663 if(new_var){
664 // Map the HDF5 dataset attributes to DAP4
665 map_h5_attrs_to_dap4(dset_id,nullptr,new_var,nullptr,1);
666 // If this variable is a hardlink, stores the HARDLINK info. as an attribute.
667 map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,nullptr,1);
668 }
669 delete bt;
670 bt = nullptr;
671 }
672 else {
673 // Next, deal with Array data. This 'else clause' runs to
674 // the end of the method.
675 auto ar = new HDF5Array(newvarname, filename, bt);
676 delete bt; bt = nullptr;
677
678 // set number of elements and variable name values.
679 // This essentially stores in the struct.
680 ar->set_memneed(dt_inst.need);
681 ar->set_numdim(dt_inst.ndims);
682 ar->set_numelm((int) (dt_inst.nelmts));
683 ar->set_varpath(varname);
684
685
686 // If we have dimension names(dimension scale is used.),we will see if we can add the names.
687 int dimnames_size = 0;
688 if((unsigned int)((int)(dt_inst.dimnames.size())) != dt_inst.dimnames.size())
689 {
690 delete ar;
691 throw
692 InternalErr(__FILE__, __LINE__,
693 "number of dimensions: overflow");
694 }
695 dimnames_size = (int)(dt_inst.dimnames.size());
696#if 0
697//cerr<<"dimnames_size is "<<dimnames_size <<endl;
698//cerr<<"ndims is "<<dt_inst.ndims <<endl;
699#endif
700
701 if(dimnames_size ==dt_inst.ndims) {
702
703 for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++) {
704 if(dt_inst.dimnames[dim_index] !="")
705 ar->append_dim(dt_inst.size[dim_index],dt_inst.dimnames[dim_index]);
706 else
707 ar->append_dim(dt_inst.size[dim_index]);
708 // D4dimension has to have a name. If no name, no D4dimension(from comments libdap4: Array.cc)
709 }
710 dt_inst.dimnames.clear();
711 }
712 else {
713 // For DAP4, no need to add dimension if no dimension name
714 for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++)
715 ar->append_dim(dt_inst.size[dim_index]);
716 }
717
718 // We need to transform dimension info. to DAP4 group
719 BaseType* new_var = nullptr;
720 try {
721 new_var = ar->h5dims_transform_to_dap4(d4_grp,dt_inst.dimnames_path);
722 }
723 catch(...) {
724 delete ar;
725 throw;
726 }
727
728 // clear DAP4 dimnames_path vector
729 dt_inst.dimnames_path.clear();
730
731 // Map HDF5 dataset attributes to DAP4
732 map_h5_attrs_to_dap4(dset_id,nullptr,new_var,nullptr,1);
733
734 // If this is a hardlink, map the Hardlink info. as an DAP4 attribute.
735 map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,nullptr,1);
736#if 0
737 // Test the attribute
738 D4Attribute *test_attr = new D4Attribute("DAP4_test",attr_str_c);
739 test_attr->add_value("test_grp_attr");
740 new_var->attributes()->add_attribute_nocopy(test_attr);
741#endif
742 // Add this var to DAP4 group.
743 d4_grp->add_var_nocopy(new_var);
744 delete ar; ar = nullptr;
745 }
746 BESDEBUG("h5", "<read_objects_base_type(dmr)" << endl);
747
748}
749
763void
764read_objects_structure(D4Group *d4_grp, const string & varname,
765 const string & filename,hid_t dset_id)
766{
767 // Obtain the relative path of the variable name under the leaf group
768 string newvarname = HDF5CFUtil::obtain_string_after_lastslash(varname);
769
770 // Map HDF5 compound datatype to Structure
771 Structure *structure = Get_structure(newvarname, varname,filename, dt_inst.type,true);
772
773 try {
774 BESDEBUG("h5", "=read_objects_structure(): Dimension is "
775 << dt_inst.ndims << endl);
776
777 if (dt_inst.ndims != 0) { // Array of Structure
778 BESDEBUG("h5", "=read_objects_structure(): array of size " <<
779 dt_inst.nelmts << endl);
780 BESDEBUG("h5", "=read_objects_structure(): memory needed = " <<
781 dt_inst.need << endl);
782
783 // Create the Array of structure.
784 auto ar = new HDF5Array(newvarname, filename, structure);
785 delete structure; structure = nullptr;
786
787
788 // These parameters are used in the data read function.
789 ar->set_memneed(dt_inst.need);
790 ar->set_numdim(dt_inst.ndims);
791 ar->set_numelm((int) (dt_inst.nelmts));
792 ar->set_length((int) (dt_inst.nelmts));
793 ar->set_varpath(varname);
794
795 // If having dimension names, add the dimension names to DAP.
796 int dimnames_size = 0;
797 if((unsigned int)((int)(dt_inst.dimnames.size())) != dt_inst.dimnames.size())
798 {
799 delete ar;
800 throw
801 InternalErr(__FILE__, __LINE__,
802 "number of dimensions: overflow");
803 }
804 dimnames_size = (int)(dt_inst.dimnames.size());
805
806
807 if(dimnames_size ==dt_inst.ndims) {
808 for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++) {
809 if(dt_inst.dimnames[dim_index] !="")
810 ar->append_dim(dt_inst.size[dim_index],dt_inst.dimnames[dim_index]);
811 else
812 ar->append_dim(dt_inst.size[dim_index]);
813 }
814 dt_inst.dimnames.clear();
815 }
816 else {
817 for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++)
818 ar->append_dim(dt_inst.size[dim_index]);
819
820 }
821
822 // We need to transform dimension info. to DAP4 group
823 BaseType* new_var = ar->h5dims_transform_to_dap4(d4_grp,dt_inst.dimnames_path);
824 dt_inst.dimnames_path.clear();
825
826 // Map HDF5 dataset attributes to DAP4
827 map_h5_attrs_to_dap4(dset_id,nullptr,new_var,nullptr,1);
828
829 // If this is a hardlink, map the Hardlink info. as an DAP4 attribute.
830 map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,nullptr,1);
831
832 // Add this var to DAP4 group
833 if(new_var)
834 d4_grp->add_var_nocopy(new_var);
835 delete ar; ar = nullptr;
836 }// end if
837 else {// A scalar structure
838
839 structure->set_is_dap4(true);
840 map_h5_attrs_to_dap4(dset_id,nullptr,nullptr,structure,2);
841 map_h5_dset_hardlink_to_d4(dset_id,varname,nullptr,structure,2);
842 if(structure)
843 d4_grp->add_var_nocopy(structure);
844 }
845 } // try Structure
846 catch (...) {
847 delete structure;
848 throw;
849 }
850}
851
852
866//
867
868void map_h5_attrs_to_dap4(hid_t h5_objid,D4Group* d4g,BaseType* d4b,Structure * d4s,int flag) {
869
870 // Get the object info
871 H5O_info_t obj_info;
872 if (H5OGET_INFO(h5_objid, &obj_info) <0) {
873 string msg = "Fail to obtain the HDF5 object info. .";
874 throw InternalErr(__FILE__, __LINE__, msg);
875 }
876
877 // Obtain the number of attributes
878 auto num_attr = (int)(obj_info.num_attrs);
879 if (num_attr < 0 ) {
880 string msg = "Fail to get the number of attributes for the HDF5 object. ";
881 throw InternalErr(__FILE__, __LINE__,msg);
882 }
883
884 string print_rep;
885 vector<char>temp_buf;
886
887 bool ignore_attr = false;
888 hid_t attr_id = -1;
889 for (int j = 0; j < num_attr; j++) {
890
891 // Obtain attribute information.
892 DSattr_t attr_inst;
893
894 // Ignore the attributes of which the HDF5 datatype
895 // cannot be mapped to DAP4. The ignored attribute datatypes can be found
896 // at function get_attr_info in h5get.cc.
897 attr_id = get_attr_info(h5_objid, j, true,&attr_inst, &ignore_attr);
898 if (true == ignore_attr) {
899 H5Aclose(attr_id);
900 continue;
901 }
902
903 // Get the corresponding DAP data type of the HDF5 datatype.
904 // The following line doesn't work in HDF5 1.10.
905#if 0
906 //hid_t ty_id = attr_inst.type;
907#endif
908 hid_t ty_id = H5Aget_type(attr_id);
909 if(ty_id <0) {
910 H5Aclose(attr_id);
911 throw InternalErr(__FILE__, __LINE__, "Cannot retrieve HDF5 attribute datatype successfully.");
912 }
913
914 string dap_type = get_dap_type(ty_id,true);
915
916 // Need to have DAP4 representation of the attribute type
917 D4AttributeType dap4_attr_type = daptype_strrep_to_dap4_attrtype(dap_type);
918
919 // We encounter an unsupported DAP4 attribute type.
920 if(attr_null_c == dap4_attr_type) {
921 H5Tclose(ty_id);
922 H5Aclose(attr_id);
923 throw InternalErr(__FILE__, __LINE__, "unsupported DAP4 attribute type");
924 }
925
926 string attr_name = attr_inst.name;
927 BESDEBUG("h5", "arttr_name= " << attr_name << endl);
928
929 // Create the DAP4 attribute mapped from HDF5
930 auto d4_attr = new D4Attribute(attr_name,dap4_attr_type);
931
932 // We have to handle variable length string differently.
933 if (H5Tis_variable_str(ty_id))
934 write_vlen_str_attrs(attr_id,ty_id,&attr_inst,d4_attr,nullptr,true);
935 else {
936
937 vector<char> value;
938#if 0
939 //value.resize(attr_inst.need + sizeof(char));
940#endif
941 value.resize(attr_inst.need);
942 BESDEBUG("h5", "arttr_inst.need=" << attr_inst.need << endl);
943
944 // Need to obtain the memtype since we still find BE data.
945 hid_t memtype = H5Tget_native_type(ty_id, H5T_DIR_ASCEND);
946 // Read HDF5 attribute data.
947 if (H5Aread(attr_id, memtype, (void *) (value.data())) < 0) {
948 delete d4_attr;
949 throw InternalErr(__FILE__, __LINE__, "unable to read HDF5 attribute data");
950 }
951 H5Aclose(memtype);
952
953 // For scalar data, just read data once.
954 if (attr_inst.ndims == 0) {
955 for (int loc = 0; loc < (int) attr_inst.nelmts; loc++) {
956 print_rep = print_attr(ty_id, loc, value.data());
957 if (print_rep.c_str() != nullptr) {
958 d4_attr->add_value(print_rep);
959 }
960 }
961
962 }
963 else {// The number of dimensions is > 0
964
965 // Get the attribute datatype size
966 int elesize = (int) H5Tget_size(ty_id);
967 if (elesize == 0) {
968 H5Tclose(ty_id);
969 H5Aclose(attr_id);
970 delete d4_attr;
971 throw InternalErr(__FILE__, __LINE__, "unable to get attibute size");
972 }
973
974 // Due to the implementation of print_attr, the attribute value will be
975 // written one by one.
976 char *tempvalue = value.data();
977
978 // Write this value. the "loc" can always be set to 0 since
979 // tempvalue will be moved to the next value.
980 for( hsize_t temp_index = 0; temp_index < attr_inst.nelmts; temp_index ++) {
981 print_rep = print_attr(ty_id, 0, tempvalue);
982 if (print_rep.c_str() != nullptr) {
983
984 BESDEBUG("h5", "print_rep= " << print_rep << endl);
985
986 d4_attr->add_value(print_rep);
987 tempvalue = tempvalue + elesize;
988 BESDEBUG("h5",
989 "tempvalue= " << tempvalue
990 << "elesize=" << elesize
991 << endl);
992
993 }
994 else {
995 H5Tclose(ty_id);
996 H5Aclose(attr_id);
997 delete d4_attr;
998 throw InternalErr(__FILE__, __LINE__, "unable to convert attibute value to DAP");
999 }
1000 }//for(hsize_t temp_index=0; .....
1001 } // if attr_inst.ndims != 0
1002 }
1003 if(H5Tclose(ty_id) < 0) {
1004 H5Aclose(attr_id);
1005 delete d4_attr;
1006 throw InternalErr(__FILE__, __LINE__, "unable to close HDF5 type id");
1007 }
1008 if (H5Aclose(attr_id) < 0) {
1009 delete d4_attr;
1010 throw InternalErr(__FILE__, __LINE__, "unable to close attibute id");
1011 }
1012
1013 if(0 == flag) // D4group
1014 d4g->attributes()->add_attribute_nocopy(d4_attr);
1015 else if (1 == flag) // HDF5 dataset with atomic datatypes
1016 d4b->attributes()->add_attribute_nocopy(d4_attr);
1017 else if ( 2 == flag) // HDF5 dataset with compound datatype
1018 d4s->attributes()->add_attribute_nocopy(d4_attr);
1019 else {
1020 stringstream sflag;
1021 sflag << flag;
1022 string msg ="The add_dap4_attr flag has to be either 0,1 or 2.";
1023 msg+="The current flag is "+sflag.str();
1024 delete d4_attr;
1025 throw InternalErr(__FILE__, __LINE__, msg);
1026 }
1027 } // for (int j = 0; j < num_attr; j++)
1028
1029 return;
1030}
1031
1045
1046
1047void map_h5_dset_hardlink_to_d4(hid_t h5_dsetid,const string & full_path, BaseType* d4b,Structure * d4s,int flag) {
1048
1049 // Obtain the unique object number info. If no hardlinks, empty string will return.
1050 string oid = get_hardlink_dmr(h5_dsetid, full_path);
1051
1052 // Find that this is a hardlink,add the hardlink info to a DAP4 attribute.
1053 if(false == oid.empty()) {
1054
1055 D4Attribute *d4_hlinfo = new D4Attribute("HDF5_HARDLINK",attr_str_c);
1056 d4_hlinfo->add_value(obj_paths.get_name(oid));
1057
1058 if (1 == flag)
1059 d4b->attributes()->add_attribute_nocopy(d4_hlinfo);
1060 else if ( 2 == flag)
1061 d4s->attributes()->add_attribute_nocopy(d4_hlinfo);
1062 else
1063 delete d4_hlinfo;
1064 }
1065
1066}
1067
1080void get_softlink(D4Group* par_grp, hid_t h5obj_id, const string & oname, int index, size_t val_size)
1081{
1082 BESDEBUG("h5", "dap4 >get_softlink():" << oname << endl);
1083
1084 ostringstream oss;
1085 oss << string("HDF5_SOFTLINK");
1086 oss << "_";
1087 oss << index;
1088 string temp_varname = oss.str();
1089
1090
1091 BESDEBUG("h5", "dap4->get_softlink():" << temp_varname << endl);
1092 auto d4_slinfo = new D4Attribute;
1093 d4_slinfo->set_name(temp_varname);
1094
1095 // Make the type as a container
1096 d4_slinfo->set_type(attr_container_c);
1097
1098 string softlink_name = "linkname";
1099
1100 auto softlink_src = new D4Attribute(softlink_name,attr_str_c);
1101 softlink_src->add_value(oname);
1102
1103 d4_slinfo->attributes()->add_attribute_nocopy(softlink_src);
1104 string softlink_value_name ="LINKTARGET";
1105
1106 // Get the link target information. We always return the link value in a string format.
1107 D4Attribute *softlink_tgt = nullptr;
1108
1109 try {
1110 vector<char> buf;
1111 buf.resize(val_size + 1);
1112
1113 // get link target name
1114 if (H5Lget_val(h5obj_id, oname.c_str(), (void*) buf.data(), val_size + 1, H5P_DEFAULT) < 0) {
1115 throw InternalErr(__FILE__, __LINE__, "unable to get link value");
1116 }
1117 softlink_tgt = new D4Attribute(softlink_value_name, attr_str_c);
1118 auto link_target_name = string(buf.begin(), buf.end());
1119 softlink_tgt->add_value(link_target_name);
1120
1121 d4_slinfo->attributes()->add_attribute_nocopy(softlink_tgt);
1122 }
1123 catch (...) {
1124 delete softlink_tgt;
1125 throw;
1126 }
1127
1128 par_grp->attributes()->add_attribute_nocopy(d4_slinfo);
1129}
1130
1131
1144string get_hardlink_dmr( hid_t h5obj_id, const string & oname) {
1145
1146 BESDEBUG("h5", "dap4->get_hardlink_dmr():" << oname << endl);
1147
1148 // Get the object info
1149 H5O_info_t obj_info;
1150 if (H5OGET_INFO(h5obj_id, &obj_info) <0) {
1151 throw InternalErr(__FILE__, __LINE__, "H5OGET_INFO() failed.");
1152 }
1153
1154 // If the reference count is greater than 1,that means
1155 // hard links are found. return the original object name this
1156 // hard link points to.
1157
1158 if (obj_info.rc >1) {
1159
1160 string objno;
1161
1162#if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
1163 char *obj_tok_str = nullptr;
1164 if(H5Otoken_to_str(h5obj_id, &(obj_info.token), &obj_tok_str) <0) {
1165 throw InternalErr(__FILE__, __LINE__, "H5Otoken_to_str failed.");
1166 }
1167 objno.assign(obj_tok_str,obj_tok_str+strlen(obj_tok_str));
1168 H5free_memory(obj_tok_str);
1169
1170#else
1171 ostringstream oss;
1172 oss << hex << obj_info.addr;
1173 objno = oss.str();
1174#endif
1175
1176 BESDEBUG("h5", "dap4->get_hardlink_dmr() objno=" << objno << endl);
1177
1178 // Add this hard link to the map.
1179 // obj_paths is a global variable defined at the beginning of this file.
1180 // it is essentially a id to obj name map. See HDF5PathFinder.h.
1181 if (!obj_paths.add(objno, oname)) {
1182 return objno;
1183 }
1184 else {
1185 return "";
1186 }
1187 }
1188 else {
1189 return "";
1190 }
1191
1192}
A class for handling all types of array in HDF5 for the default option.
This class provides a way to map HDF5 byte to DAP Byte for the default option.
This file includes several helper functions for translating HDF5 to CF-compliant.
A class for mapping HDF5 32-bit float to DAP for the default option.
A class for mapping HDF5 64-bit float to DAP for the default option.
A class for HDF5 signed 16 bit integer type.
This class provides a way to map HDF5 32 bit integer to DAP Int32 for the default option.
This class that translates HDF5 string into DAP string for the default option.
This class converts HDF5 compound type into DAP structure for the default option.
This class provides a way to map unsigned HDF5 16 bit integer to DAP UInt16 for the default option.
This class provides a way to map unsigned HDF5 32 bit integer to DAP UInt32.
This class generates DAP URL type for the default option.
bool add(const std::string &id, const std::string &name)
std::string get_name(const std::string &id)
void depth_first(hid_t pid, const char *gname, DAS &das)
Definition: h5das.cc:62
void read_objects(DAS &das, const string &varname, hid_t oid, int num_attr)
Definition: h5das.cc:295
void read_objects_base_type(DDS &dds_table, const string &varname, const string &filename)
Definition: h5dds.cc:259
void read_objects_structure(DDS &dds_table, const string &varname, const string &filename)
Definition: h5dds.cc:313
void map_h5_attrs_to_dap4(hid_t oid, D4Group *d4g, BaseType *d4b, Structure *d4s, int flag)
A function that map HDF5 attributes to DAP4.
Definition: h5dmr.cc:868
HDF5PathFinder obj_paths
A variable for remembering visited paths to break cyclic HDF5 groups.
Definition: h5dmr.cc:68
bool breadth_first(const hid_t file_id, hid_t pid, const char *gname, D4Group *par_grp, const char *fname, bool use_dimscale, vector< link_info_t > &hdf5_hls)
Definition: h5dmr.cc:321
void get_softlink(D4Group *par_grp, hid_t h5obj_id, const string &oname, int index, size_t val_size)
Definition: h5dmr.cc:1080
Data structure and retrieval processing header for the default option.
string print_attr(hid_t type, int loc, void *sm_buf)
Definition: h5get.cc:868
string get_dap_type(hid_t type, bool is_dap4)
Definition: h5get.cc:292
void get_dataset(hid_t pid, const string &dname, DS_t *dt_inst_ptr)
Definition: h5get.cc:453
hid_t get_attr_info(hid_t dset, int index, bool is_dap4, DSattr_t *attr_inst_ptr, bool *ignore_attr_ptr)
Definition: h5get.cc:90
The main header of the HDF5 OPeNDAP handler.
const int DODS_NAMELEN
Maximum length of variable or attribute name(default option only).
Definition: hdf5_handler.h:65
A structure for DDS generation.
Definition: hdf5_handler.h:71
hsize_t nelmts
Number of elements.
Definition: hdf5_handler.h:91
hsize_t need
Space needed.
Definition: hdf5_handler.h:93
hid_t type
HDF5 data set id.
Definition: hdf5_handler.h:79
int size[DODS_MAX_RANK]
Size of each dimension.
Definition: hdf5_handler.h:87
int ndims
HDF5 data space id.
Definition: hdf5_handler.h:85
A structure for DAS generation.
Definition: hdf5_handler.h:96
char name[DODS_NAMELEN]
Name of HDF5 group or dataset.
Definition: hdf5_handler.h:98
int ndims
Number of dimensions.
Definition: hdf5_handler.h:102
hsize_t nelmts
Number of elements.
Definition: hdf5_handler.h:106
hsize_t need
Memory space needed to hold nelmts type.
Definition: hdf5_handler.h:108