bes Updated for version 3.20.13
h5get.cc
Go to the documentation of this file.
1// This file is part of hdf5_handler: an HDF5 file handler for the OPeNDAP
2// data server.
3
4// Copyright (c) 2007-2016 The HDF Group, Inc. and OPeNDAP, Inc.
5//
6// This is free software; you can redistribute it and/or modify it under the
7// terms of the GNU Lesser General Public License as published by the Free
8// Software Foundation; either version 2.1 of the License, or (at your
9// option) any later version.
10//
11// This software is distributed in the hope that it will be useful, but
12// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14// License for more details.
15//
16// You should have received a copy of the GNU Lesser General Public
17// License along with this library; if not, write to the Free Software
18// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19//
20// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
21// You can contact The HDF Group, Inc. at 1800 South Oak Street,
22// Suite 203, Champaign, IL 61820
23
36
37#include "h5get.h"
38#include "HDF5Int32.h"
39#include "HDF5UInt32.h"
40#include "HDF5UInt16.h"
41#include "HDF5Int16.h"
42#include "HDF5Byte.h"
43#include "HDF5Int8.h"
44#include "HDF5Int64.h"
45#include "HDF5UInt64.h"
46#include "HDF5Array.h"
47#include "HDF5Str.h"
48#include "HDF5Float32.h"
49#include "HDF5Float64.h"
50#include "HDF5Url.h"
51#include "HDF5Structure.h"
52#include "HDF5RequestHandler.h"
53
54#include <BESDebug.h>
55#include <math.h>
56#include <sstream>
57
58using namespace libdap;
59
60
61// H5Lvisit call back function. After finding all the hard links, return 1.
62static int visit_link_cb(hid_t group_id, const char *name, const H5L_info_t *oinfo,
63 void *_op_data);
64
65// H5OVISIT call back function. When finding the dimension scale attributes, return 1.
66static int
67visit_obj_cb(hid_t o_id, const char *name, const H5O_info_t *oinfo,
68 void *_op_data);
69
70// H5Aiterate2 call back function, check if having the dimension scale attributes.
71static herr_t attr_info_dimscale(hid_t loc_id, const char *name, const H5A_info_t *ainfo, void *opdata);
72
73
90hid_t get_attr_info(hid_t dset, int index, bool is_dap4, DSattr_t * attr_inst_ptr,
91 bool *ignore_attr_ptr)
92{
93
94 hid_t attrid = -1;
95
96 // Always assume that we don't ignore any attributes.
97 *ignore_attr_ptr = false;
98
99 if ((attrid = H5Aopen_by_idx(dset, ".", H5_INDEX_CRT_ORDER, H5_ITER_INC,(hsize_t)index, H5P_DEFAULT, H5P_DEFAULT)) < 0) {
100 string msg = "unable to open attribute by index ";
101 throw InternalErr(__FILE__, __LINE__, msg);
102 }
103
104 // Obtain the size of attribute name.
105 ssize_t name_size = H5Aget_name(attrid, 0, nullptr);
106 if (name_size < 0) {
107 H5Aclose(attrid);
108 string msg = "unable to obtain the size of the hdf5 attribute name ";
109 throw InternalErr(__FILE__, __LINE__, msg);
110 }
111
112 vector<char> attr_name;
113 attr_name.resize(name_size+1);
114 // Obtain the attribute name.
115 if ((H5Aget_name(attrid, name_size+1, attr_name.data())) < 0) {
116 H5Aclose(attrid);
117 string msg = "unable to obtain the hdf5 attribute name ";
118 throw InternalErr(__FILE__, __LINE__, msg);
119 }
120
121 // Obtain the type of the attribute.
122 hid_t ty_id = -1;
123 if ((ty_id = H5Aget_type(attrid)) < 0) {
124 string msg = "unable to obtain hdf5 datatype for the attribute ";
125 string attrnamestr(attr_name.begin(),attr_name.end());
126 msg += attrnamestr;
127 H5Aclose(attrid);
128 throw InternalErr(__FILE__, __LINE__, msg);
129 }
130
131 H5T_class_t ty_class = H5Tget_class(ty_id);
132 if (ty_class < 0) {
133 string msg = "cannot get hdf5 attribute datatype class for the attribute ";
134 string attrnamestr(attr_name.begin(),attr_name.end());
135 msg += attrnamestr;
136 H5Aclose(attrid);
137 throw InternalErr(__FILE__, __LINE__, msg);
138 }
139
140 // The following datatype will not be supported for mapping to DAS for both DAP2 and DAP4.
141 // Note:DAP4 explicitly states that the data should be defined atomic datatype(int, string).
142 // 1-D variable length of string can also be mapped to both DAS and DDS.
143 // The variable length string class is H5T_STRING rather than H5T_VLEN,
144 // So safe here.
145 // We also ignore the mapping of integer 64 bit for DAP2 since DAP2 doesn't
146 // support 64-bit integer. In theory, DAP2 doesn't support long double
147 // (128-bit or 92-bit floating point type), since this rarely happens
148 // in DAP application, we simply don't consider here.
149 // However, DAP4 supports 64-bit integer.
150 if ((ty_class == H5T_TIME) || (ty_class == H5T_BITFIELD)
151 || (ty_class == H5T_OPAQUE) || (ty_class == H5T_ENUM)
152 || (ty_class == H5T_REFERENCE) ||(ty_class == H5T_COMPOUND)
153 || (ty_class == H5T_VLEN) || (ty_class == H5T_ARRAY)){
154
155 *ignore_attr_ptr = true;
156 H5Tclose(ty_id);
157 return attrid;
158 }
159
160 // Ignore 64-bit integer for DAP2.
161 // The nested if is better to understand the code. Don't combine
162 if (false == is_dap4) {
163 if((ty_class == H5T_INTEGER) && (H5Tget_size(ty_id)== 8)) {//64-bit int
164 *ignore_attr_ptr = true;
165 H5Tclose(ty_id);
166 return attrid;
167 }
168 }
169
170 // Here we ignore netCDF-4 specific attributes for DAP4 to make filenetCDF-4 work
171 if (true == is_dap4 && HDF5RequestHandler::get_default_handle_dimension() == true) {
172 // Remove the nullptrTERM etc.
173 string attr_name_str(attr_name.begin(),attr_name.end()-1);
174 if(attr_name_str == "CLASS" || attr_name_str == "NAME" || attr_name_str == "_Netcdf4Dimid"
175 || attr_name_str == "_nc3_strict" || attr_name_str=="_NCProperties" || attr_name_str=="_Netcdf4Coordinates") {
176 *ignore_attr_ptr = true;
177 H5Tclose(ty_id);
178 return attrid;
179 }
180 }
181
182 hid_t aspace_id = -1;
183 if ((aspace_id = H5Aget_space(attrid)) < 0) {
184 string msg = "cannot get hdf5 dataspace id for the attribute ";
185 string attrnamestr(attr_name.begin(),attr_name.end());
186 msg += attrnamestr;
187 H5Aclose(attrid);
188 throw InternalErr(__FILE__, __LINE__, msg);
189 }
190
191 // It is better to use the dynamic allocation of the array.
192 // However, since the DODS_MAX_RANK is not big and it is also
193 // used in other location, we still keep the original code.
194 // KY 2011-11-16
195
196 int ndims = H5Sget_simple_extent_ndims(aspace_id);
197 if (ndims < 0) {
198 string msg = "cannot get hdf5 dataspace number of dimension for attribute ";
199 string attrnamestr(attr_name.begin(),attr_name.end());
200 msg += attrnamestr;
201 H5Sclose(aspace_id);
202 H5Aclose(attrid);
203 throw InternalErr(__FILE__, __LINE__, msg);
204 }
205
206 // Check if the dimension size exceeds the maximum number of dimension DAP supports
207 if (ndims > DODS_MAX_RANK) {
208 string msg = "number of dimensions exceeds allowed for attribute ";
209 string attrnamestr(attr_name.begin(),attr_name.end());
210 msg += attrnamestr;
211 H5Sclose(aspace_id);
212 H5Aclose(attrid);
213 throw InternalErr(__FILE__, __LINE__, msg);
214 }
215
216 vector<hsize_t> size(ndims);
217 vector<hsize_t> maxsize(ndims);
218
219 //The HDF5 attribute should not have unlimited dimension,
220 // maxsize is only a place holder.
221 if (H5Sget_simple_extent_dims(aspace_id, size.data(), maxsize.data())<0){
222 string msg = "cannot obtain the dim. info for the attribute ";
223 string attrnamestr(attr_name.begin(),attr_name.end());
224 msg += attrnamestr;
225 H5Sclose(aspace_id);
226 H5Aclose(attrid);
227 throw InternalErr(__FILE__, __LINE__, msg);
228 }
229
230 // Return ndims and size[ndims].
231 hsize_t nelmts = 1;
232 if (ndims) {
233 for (int j = 0; j < ndims; j++)
234 nelmts *= size[j];
235 }
236
237 size_t ty_size = H5Tget_size(ty_id);
238 if (ty_size == 0) {
239 string msg = "cannot obtain the dtype size for the attribute ";
240 string attrnamestr(attr_name.begin(),attr_name.end());
241 msg += attrnamestr;
242 H5Sclose(aspace_id);
243 H5Aclose(attrid);
244 throw InternalErr(__FILE__, __LINE__, msg);
245 }
246
247 size_t need = nelmts * H5Tget_size(ty_id);
248
249 // We want to save memory type in the struct.
250 hid_t memtype = H5Tget_native_type(ty_id, H5T_DIR_ASCEND);
251 if (memtype < 0){
252 string msg = "cannot obtain the memory dtype for the attribute ";
253 string attrnamestr(attr_name.begin(),attr_name.end());
254 msg += attrnamestr;
255 H5Sclose(aspace_id);
256 H5Aclose(attrid);
257 throw InternalErr(__FILE__, __LINE__, msg);
258 }
259
260 // Save the information to the struct
261 (*attr_inst_ptr).type = memtype;
262 (*attr_inst_ptr).ndims = ndims;
263 (*attr_inst_ptr).nelmts = nelmts;
264 (*attr_inst_ptr).need = need;
265 strncpy((*attr_inst_ptr).name, attr_name.data(), name_size+1);
266
267 // The handler assumes the size of an attribute is limited to 32-bit int
268 for (int j = 0; j < ndims; j++) {
269 (*attr_inst_ptr).size[j] = (int)(size[j]);
270 }
271
272 if(H5Sclose(aspace_id)<0) {
273 H5Aclose(attrid);
274 throw InternalErr(__FILE__,__LINE__,"Cannot close HDF5 dataspace ");
275 }
276
277 return attrid;
278}
279
292string get_dap_type(hid_t type, bool is_dap4)
293{
294 size_t size = 0;
295 H5T_sign_t sign;
296 BESDEBUG("h5", ">get_dap_type(): type=" << type << endl);
297 H5T_class_t class_t = H5Tget_class(type);
298 if (H5T_NO_CLASS == class_t)
299 throw InternalErr(__FILE__, __LINE__,
300 "The HDF5 datatype doesn't belong to any Class.");
301 switch (class_t) {
302
303 case H5T_INTEGER:
304
305 size = H5Tget_size(type);
306 if (size == 0){
307 throw InternalErr(__FILE__, __LINE__,
308 "size of datatype is invalid");
309 }
310
311 sign = H5Tget_sign(type);
312 if (sign < 0){
313 throw InternalErr(__FILE__, __LINE__,
314 "sign of datatype is invalid");
315 }
316
317 BESDEBUG("h5", "=get_dap_type(): H5T_INTEGER" <<
318 " sign = " << sign <<
319 " size = " << size <<
320 endl);
321 if (size == 1){
322 // DAP2 doesn't have signed 8-bit integer, so we need map it to INT16.
323 if(true == is_dap4) {
324 if (sign == H5T_SGN_NONE)
325 return BYTE;
326 else
327 return INT8;
328 }
329 else {
330 if (sign == H5T_SGN_NONE)
331 return BYTE;
332 else
333 return INT16;
334 }
335 }
336
337 if (size == 2) {
338 if (sign == H5T_SGN_NONE)
339 return UINT16;
340 else
341 return INT16;
342 }
343
344 if (size == 4) {
345 if (sign == H5T_SGN_NONE)
346 return UINT32;
347 else
348 return INT32;
349 }
350
351 if(size == 8) {
352 // DAP4 supports 64-bit integer.
353 if (true == is_dap4) {
354 if (sign == H5T_SGN_NONE)
355 return UINT64;
356 else
357 return INT64;
358 }
359 else
360 return INT_ELSE;
361 }
362
363 return INT_ELSE;
364
365 case H5T_FLOAT:
366 size = H5Tget_size(type);
367 if (size == 0){
368 throw InternalErr(__FILE__, __LINE__,
369 "size of the datatype is invalid");
370 }
371
372 BESDEBUG("h5", "=get_dap_type(): FLOAT size = " << size << endl);
373 if (size == 4)
374 return FLOAT32;
375 if (size == 8)
376 return FLOAT64;
377
378 return FLOAT_ELSE;
379
380 case H5T_STRING:
381 BESDEBUG("h5", "<get_dap_type(): H5T_STRING" << endl);
382 return STRING;
383
384 case H5T_REFERENCE:
385 BESDEBUG("h5", "<get_dap_type(): H5T_REFERENCE" << endl);
386 return URL;
387 // Note: Currently DAP2 and DAP4 only support defined atomic types.
388 // So the H5T_COMPOUND and H5_ARRAY cases should never be reached for attribute handling.
389 // However, this function may be used for variable handling.
390 case H5T_COMPOUND:
391 BESDEBUG("h5", "<get_dap_type(): COMPOUND" << endl);
392 return COMPOUND;
393
394 case H5T_ARRAY:
395 return ARRAY;
396
397 default:
398 BESDEBUG("h5", "<get_dap_type(): Unmappable Type" << endl);
399 return "Unmappable Type";
400 }
401}
402
412hid_t get_fileid(const char *filename)
413{
414 hid_t fileid = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT);
415 if (fileid < 0){
416 string msg = "cannot open the HDF5 file ";
417 string filenamestr(filename);
418 msg += filenamestr;
419 throw InternalErr(__FILE__, __LINE__, msg);
420 }
421
422 return fileid;
423}
424
434void close_fileid(hid_t fid)
435{
436 if (H5Fclose(fid) < 0)
437 throw Error(unknown_error,
438 string("Could not close the HDF5 file."));
439
440}
441
452
453void get_dataset(hid_t pid, const string &dname, DS_t * dt_inst_ptr)
454{
455 BESDEBUG("h5", ">get_dataset()" << endl);
456
457 // Obtain the dataset ID
458 hid_t dset = -1;
459 if ((dset = H5Dopen(pid, dname.c_str(),H5P_DEFAULT)) < 0) {
460 string msg = "cannot open the HDF5 dataset ";
461 msg += dname;
462 throw InternalErr(__FILE__, __LINE__, msg);
463 }
464
465 // Obtain the datatype ID
466 hid_t dtype = -1;
467 if ((dtype = H5Dget_type(dset)) < 0) {
468 H5Dclose(dset);
469 string msg = "cannot get the the datatype of HDF5 dataset ";
470 msg += dname;
471 throw InternalErr(__FILE__, __LINE__, msg);
472 }
473
474 // Obtain the datatype class
475 H5T_class_t ty_class = H5Tget_class(dtype);
476 if (ty_class < 0) {
477 H5Tclose(dtype);
478 H5Dclose(dset);
479 string msg = "cannot get the datatype class of HDF5 dataset ";
480 msg += dname;
481 throw InternalErr(__FILE__, __LINE__, msg);
482 }
483
484 // These datatype classes are unsupported. Note we do support
485 // variable length string and the variable length string class is
486 // H5T_STRING rather than H5T_VLEN.
487 if ((ty_class == H5T_TIME) || (ty_class == H5T_BITFIELD)
488 || (ty_class == H5T_OPAQUE) || (ty_class == H5T_ENUM) || (ty_class == H5T_VLEN)) {
489 string msg = "unexpected datatype of HDF5 dataset ";
490 msg += dname;
491 throw InternalErr(__FILE__, __LINE__, msg);
492 }
493
494 hid_t dspace = -1;
495 if ((dspace = H5Dget_space(dset)) < 0) {
496 H5Tclose(dtype);
497 H5Dclose(dset);
498 string msg = "cannot get the the dataspace of HDF5 dataset ";
499 msg += dname;
500 throw InternalErr(__FILE__, __LINE__, msg);
501 }
502
503 // It is better to use the dynamic allocation of the array.
504 // However, since the DODS_MAX_RANK is not big and it is also
505 // used in other location, we still keep the original code.
506 // KY 2011-11-17
507
508 int ndims = H5Sget_simple_extent_ndims(dspace);
509 if (ndims < 0) {
510 H5Tclose(dtype);
511 H5Sclose(dspace);
512 H5Dclose(dset);
513 string msg = "cannot get hdf5 dataspace number of dimension for dataset ";
514 msg += dname;
515 throw InternalErr(__FILE__, __LINE__, msg);
516 }
517
518 // Check if the dimension size exceeds the maximum number of dimension DAP supports
519 if (ndims > DODS_MAX_RANK) {
520 string msg = "number of dimensions exceeds allowed for dataset ";
521 msg += dname;
522 H5Tclose(dtype);
523 H5Sclose(dspace);
524 H5Dclose(dset);
525 throw InternalErr(__FILE__, __LINE__, msg);
526 }
527
528 vector<hsize_t>size(ndims);
529 vector<hsize_t>maxsize(ndims);
530
531 // Retrieve size. DAP4 doesn't have a convention to support multi-unlimited dimension yet.
532 if (H5Sget_simple_extent_dims(dspace, size.data(), maxsize.data())<0){
533 string msg = "cannot obtain the dim. info for the dataset ";
534 msg += dname;
535 H5Tclose(dtype);
536 H5Sclose(dspace);
537 H5Dclose(dset);
538 throw InternalErr(__FILE__, __LINE__, msg);
539 }
540
541 // return ndims and size[ndims].
542 hsize_t nelmts = 1;
543 if (ndims !=0) {
544 for (int j = 0; j < ndims; j++)
545 nelmts *= size[j];
546 }
547
548 size_t dtype_size = H5Tget_size(dtype);
549 if (dtype_size == 0) {
550 string msg = "cannot obtain the data type size for the dataset ";
551 msg += dname;
552 H5Tclose(dtype);
553 H5Sclose(dspace);
554 H5Dclose(dset);
555 throw InternalErr(__FILE__, __LINE__, msg);
556 }
557
558 size_t need = nelmts * dtype_size;
559
560 hid_t memtype = H5Tget_native_type(dtype, H5T_DIR_ASCEND);
561 if (memtype < 0){
562 string msg = "cannot obtain the memory data type for the dataset ";
563 msg += dname;
564 H5Tclose(dtype);
565 H5Sclose(dspace);
566 H5Dclose(dset);
567 throw InternalErr(__FILE__, __LINE__, msg);
568 }
569
570 (*dt_inst_ptr).type = memtype;
571 (*dt_inst_ptr).ndims = ndims;
572 (*dt_inst_ptr).nelmts = nelmts;
573 (*dt_inst_ptr).need = need;
574 strncpy((*dt_inst_ptr).name, dname.c_str(), dname.length());
575 (*dt_inst_ptr).name[dname.length()] = '\0';
576 for (int j = 0; j < ndims; j++)
577 (*dt_inst_ptr).size[j] = (int)(size[j]);
578
579 if(H5Tclose(dtype)<0) {
580 H5Sclose(dspace);
581 H5Dclose(dset);
582 throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 datatype.");
583 }
584
585 if(H5Sclose(dspace)<0) {
586 H5Dclose(dset);
587 throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 dataspace.");
588 }
589
590 if(H5Dclose(dset)<0) {
591 throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 dataset.");
592 }
593
594}
595
610void get_dataset_dmr(const hid_t file_id, hid_t pid, const string &dname, DS_t * dt_inst_ptr,bool use_dimscale, bool &is_pure_dim, vector<link_info_t> &hdf5_hls)
611{
612
613 BESDEBUG("h5", ">get_dataset()" << endl);
614
615 // Obtain the dataset ID
616 hid_t dset = -1;
617 if ((dset = H5Dopen(pid, dname.c_str(),H5P_DEFAULT)) < 0) {
618 string msg = "cannot open the HDF5 dataset ";
619 msg += dname;
620 throw InternalErr(__FILE__, __LINE__, msg);
621 }
622
623 // Obtain the datatype ID
624 hid_t dtype = -1;
625 if ((dtype = H5Dget_type(dset)) < 0) {
626 H5Dclose(dset);
627 string msg = "cannot get the the datatype of HDF5 dataset ";
628 msg += dname;
629 throw InternalErr(__FILE__, __LINE__, msg);
630 }
631
632 // Obtain the datatype class
633 H5T_class_t ty_class = H5Tget_class(dtype);
634 if (ty_class < 0) {
635 H5Tclose(dtype);
636 H5Dclose(dset);
637 string msg = "cannot get the datatype class of HDF5 dataset ";
638 msg += dname;
639 throw InternalErr(__FILE__, __LINE__, msg);
640 }
641
642 // These datatype classes are unsupported. Note we do support
643 // variable length string and the variable length string class is
644 // H5T_STRING rather than H5T_VLEN.
645 if ((ty_class == H5T_TIME) || (ty_class == H5T_BITFIELD)
646 || (ty_class == H5T_OPAQUE) || (ty_class == H5T_ENUM) || (ty_class == H5T_VLEN)) {
647 string msg = "unexpected datatype of HDF5 dataset ";
648 msg += dname;
649 throw InternalErr(__FILE__, __LINE__, msg);
650 }
651
652 hid_t dspace = -1;
653 if ((dspace = H5Dget_space(dset)) < 0) {
654 H5Tclose(dtype);
655 H5Dclose(dset);
656 string msg = "cannot get the the dataspace of HDF5 dataset ";
657 msg += dname;
658 throw InternalErr(__FILE__, __LINE__, msg);
659 }
660
661 // It is better to use the dynamic allocation of the array.
662 // However, since the DODS_MAX_RANK is not big and it is also
663 // used in other location, we still keep the original code.
664 // KY 2011-11-17
665
666 int ndims = H5Sget_simple_extent_ndims(dspace);
667 if (ndims < 0) {
668 H5Tclose(dtype);
669 H5Sclose(dspace);
670 H5Dclose(dset);
671 string msg = "cannot get hdf5 dataspace number of dimension for dataset ";
672 msg += dname;
673 throw InternalErr(__FILE__, __LINE__, msg);
674 }
675
676 // Check if the dimension size exceeds the maximum number of dimension DAP supports
677 if (ndims > DODS_MAX_RANK) {
678 string msg = "number of dimensions exceeds allowed for dataset ";
679 msg += dname;
680 H5Tclose(dtype);
681 H5Sclose(dspace);
682 H5Dclose(dset);
683 throw InternalErr(__FILE__, __LINE__, msg);
684 }
685
686 vector<hsize_t>size(ndims);
687 vector<hsize_t>maxsize(ndims);
688
689 // Retrieve size. DAP4 doesn't have a convention to support multi-unlimited dimension yet.
690 if (H5Sget_simple_extent_dims(dspace, size.data(), maxsize.data())<0){
691 string msg = "cannot obtain the dim. info for the dataset ";
692 msg += dname;
693 H5Tclose(dtype);
694 H5Sclose(dspace);
695 H5Dclose(dset);
696 throw InternalErr(__FILE__, __LINE__, msg);
697 }
698
699 // return ndims and size[ndims].
700 hsize_t nelmts = 1;
701 if (ndims !=0) {
702 for (int j = 0; j < ndims; j++)
703 nelmts *= size[j];
704 }
705
706 size_t dtype_size = H5Tget_size(dtype);
707 if (dtype_size == 0) {
708 string msg = "cannot obtain the data type size for the dataset ";
709 msg += dname;
710 H5Tclose(dtype);
711 H5Sclose(dspace);
712 H5Dclose(dset);
713 throw InternalErr(__FILE__, __LINE__, msg);
714 }
715
716 size_t need = nelmts * dtype_size;
717
718 hid_t memtype = H5Tget_native_type(dtype, H5T_DIR_ASCEND);
719 if (memtype < 0){
720 string msg = "cannot obtain the memory data type for the dataset ";
721 msg += dname;
722 H5Tclose(dtype);
723 H5Sclose(dspace);
724 H5Dclose(dset);
725 throw InternalErr(__FILE__, __LINE__, msg);
726 }
727
728 (*dt_inst_ptr).type = memtype;
729 (*dt_inst_ptr).ndims = ndims;
730 (*dt_inst_ptr).nelmts = nelmts;
731 (*dt_inst_ptr).need = need;
732 strncpy((*dt_inst_ptr).name, dname.c_str(), dname.length());
733 (*dt_inst_ptr).name[dname.length()] = '\0';
734 for (int j = 0; j < ndims; j++)
735 (*dt_inst_ptr).size[j] = (int)(size[j]);
736
737 // For DAP4 when dimension scales are used.
738 if(true == use_dimscale) {
739 BESDEBUG("h5", "<h5get.cc: get_dataset() use dim scale is true." << endl);
740
741 // Some HDF5 datasets are dimension scale datasets; some are not. We need to distinguish.
742 bool is_dimscale = false;
743
744 // Dimension scales must be 1-D.
745 if(1 == ndims) {
746
747 bool has_ds_attr = false;
748
749 try{
750 has_ds_attr = has_dimscale_attr(dset);
751 }
752 catch(...) {
753 H5Tclose(dtype);
754 H5Sclose(dspace);
755 H5Dclose(dset);
756 throw InternalErr(__FILE__, __LINE__, "Fail to check dim. scale.");
757 }
758
759 if(true == has_ds_attr) {
760
761#if 0
762 //int count = 0;
763 //vector<bool>dim_attr_mark;
764 //dim_attr_mark.resize(4);
765 //bool dim_attr_mark[4];
766#endif
767 int dim_attr_mark[3];
768 for(int i = 0;i<3;i++)
769 dim_attr_mark[i] = 0;
770
771 // This will check if "NAME" and "REFERENCE_LIST" exists.
772 //herr_t ret = H5Aiterate2(dset, H5_INDEX_NAME, H5_ITER_INC, nullptr, attr_info, dim_attr_mark.data());
773 herr_t ret = H5Aiterate2(dset, H5_INDEX_NAME, H5_ITER_INC, nullptr, attr_info_dimscale, dim_attr_mark);
774 if(ret < 0) {
775 string msg = "cannot interate the attributes of the dataset ";
776 msg += dname;
777 H5Tclose(dtype);
778 H5Sclose(dspace);
779 H5Dclose(dset);
780 throw InternalErr(__FILE__, __LINE__, msg);
781 }
782
783 for (int i = 0; i<3;i++)
784 BESDEBUG("h5","dim_attr_mark is "<<dim_attr_mark[i] <<endl);
785 // Find the dimension scale. DIM*SCALE is a must. Then NAME=VARIABLE or (REFERENCE_LIST and not PURE DIM)
786 // Here a little bias towards files created by the netCDF-4 APIs.
787 // If we don't have RERERENCE_LIST in a dataset that has CLASS=DIMENSION_SCALE attribute,
788 // we will ignore this orphage dimension sale since it is not associated with other datasets.
789 // However, it is an orphage dimension scale created by the netCDF-4 APIs, we think
790 // it must have a purpose to do this way by data creator. So keep this as a dimension scale.
791 //
792 if ((dim_attr_mark[0] && !dim_attr_mark[1]) || dim_attr_mark[2])
793 is_dimscale =true;
794 else if(dim_attr_mark[1])
795 is_pure_dim = true;
796 }
797 }
798
799 if (true == is_dimscale) {
800 BESDEBUG("h5", "<h5get.cc: dname is " << dname << endl);
801 BESDEBUG("h5", "<h5get.cc: get_dataset() this is dim scale." << endl);
802 BESDEBUG("h5", "<h5get.cc: dataset storage size is: " <<H5Dget_storage_size(dset)<< endl);
803#if 0
804 //if(H5Dget_storage_size(dset)!=0) {
805#endif
806 // Save the dimension names.We Only need to provide the dimension name(not the full path).
807 // We still need the dimension name fullpath for distinguishing the different dimension that
808 // has the same dimension name but in the different path
809 // TODO; pure dimension doesn't work for all cases. See https://jira.hdfgroup.org/browse/HFVHANDLER-340
810 (*dt_inst_ptr).dimnames.push_back(dname.substr(dname.find_last_of("/")+1));
811 (*dt_inst_ptr).dimnames_path.push_back(dname);
812#if 0
813 //}
814 //else
815 //is_pure_dim = true;
816#endif
817 is_pure_dim = false;
818 }
819
820 else if(false == is_pure_dim) // Except pure dimension,we need to save all dimension names in this dimension.
821 obtain_dimnames(file_id,dset,ndims,dt_inst_ptr,hdf5_hls);
822 }
823
824 if(H5Tclose(dtype)<0) {
825 H5Sclose(dspace);
826 H5Dclose(dset);
827 throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 datatype.");
828 }
829
830 if(H5Sclose(dspace)<0) {
831 H5Dclose(dset);
832 throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 dataspace.");
833 }
834
835 if(H5Dclose(dset)<0) {
836 throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 dataset.");
837 }
838
839}
840
849bool check_h5str(hid_t h5type)
850{
851 if (H5Tget_class(h5type) == H5T_STRING)
852 return true;
853 else
854 return false;
855}
856
857
868string print_attr(hid_t type, int loc, void *sm_buf) {
869 union {
870 unsigned char* ucp;
871 char *tcp;
872 short *tsp;
873 unsigned short *tusp;
874 int *tip;
875 unsigned int*tuip;
876 long *tlp;
877 unsigned long*tulp;
878 float *tfp;
879 double *tdp;
880 } gp;
881
882 vector<char> rep;
883
884 switch (H5Tget_class(type)) {
885
886 case H5T_INTEGER: {
887
888 size_t size = H5Tget_size(type);
889 if (size == 0){
890 throw InternalErr(__FILE__, __LINE__,
891 "size of datatype is invalid");
892 }
893
894 H5T_sign_t sign = H5Tget_sign(type);
895 if (sign < 0){
896 throw InternalErr(__FILE__, __LINE__,
897 "sign of datatype is invalid");
898 }
899
900 BESDEBUG("h5", "=get_dap_type(): H5T_INTEGER" <<
901 " sign = " << sign <<
902 " size = " << size <<
903 endl);
904
905 // change void pointer into the corresponding integer datatype.
906 // 32 should be long enough to hold one integer and one
907 // floating point number.
908 rep.resize(32);
909
910 if (size == 1){
911
912 if(sign == H5T_SGN_NONE) {
913 gp.ucp = (unsigned char *) sm_buf;
914 unsigned char tuchar = *(gp.ucp + loc);
915 snprintf(rep.data(), 32, "%u", tuchar);
916 }
917
918 else {
919 gp.tcp = (char *) sm_buf;
920 snprintf(rep.data(), 32, "%d", *(gp.tcp + loc));
921 }
922 }
923
924 else if (size == 2) {
925
926 if(sign == H5T_SGN_NONE) {
927 gp.tusp = (unsigned short *) sm_buf;
928 snprintf(rep.data(), 32, "%hu", *(gp.tusp + loc));
929
930 }
931 else {
932 gp.tsp = (short *) sm_buf;
933 snprintf(rep.data(), 32, "%hd", *(gp.tsp + loc));
934
935 }
936 }
937
938 else if (size == 4) {
939
940 if(sign == H5T_SGN_NONE) {
941 gp.tuip = (unsigned int *) sm_buf;
942 snprintf(rep.data(), 32, "%u", *(gp.tuip + loc));
943
944 }
945 else {
946 gp.tip = (int *) sm_buf;
947 snprintf(rep.data(), 32, "%d", *(gp.tip + loc));
948 }
949 }
950 else if (size == 8) {
951
952 if(sign == H5T_SGN_NONE) {
953 gp.tulp = (unsigned long *) sm_buf;
954 snprintf(rep.data(), 32, "%lu", *(gp.tulp + loc));
955 }
956 else {
957 gp.tlp = (long *) sm_buf;
958 snprintf(rep.data(), 32, "%ld", *(gp.tlp + loc));
959 }
960 }
961 else
962 throw InternalErr(__FILE__, __LINE__,"Unsupported integer type, check the size of datatype.");
963
964 break;
965 }
966
967 case H5T_FLOAT: {
968 rep.resize(32);
969 char gps[32];
970
971 if (H5Tget_size(type) == 4) {
972
973 float attr_val = *(float*)sm_buf;
974 bool is_a_fin = isfinite(attr_val);
975 // Represent the float number.
976 // Some space may be wasted. But it is okay.
977 gp.tfp = (float *) sm_buf;
978 int ll = snprintf(gps, 30, "%.10g", *(gp.tfp + loc));
979#if 0
980 //int ll = strlen(gps);
981#endif
982 // Add the dot to assure this is a floating number
983 if (!strchr(gps, '.') && !strchr(gps, 'e') && !strchr(gps,'E')
984 && (true == is_a_fin)){
985 gps[ll++] = '.';
986 }
987
988 gps[ll] = '\0';
989 snprintf(rep.data(), 32, "%s", gps);
990 }
991 else if (H5Tget_size(type) == 8) {
992
993 double attr_val = *(double*)sm_buf;
994 bool is_a_fin = isfinite(attr_val);
995 gp.tdp = (double *) sm_buf;
996 int ll = snprintf(gps, 30, "%.17g", *(gp.tdp + loc));
997#if 0
998 //int ll = strlen(gps);
999#endif
1000 if (!strchr(gps, '.') && !strchr(gps, 'e')&& !strchr(gps,'E')
1001 && (true == is_a_fin)) {
1002 gps[ll++] = '.';
1003 }
1004 gps[ll] = '\0';
1005 snprintf(rep.data(), 32, "%s", gps);
1006 }
1007 else if (H5Tget_size(type) == 0){
1008 throw InternalErr(__FILE__, __LINE__, "H5Tget_size() failed.");
1009 }
1010 break;
1011 }
1012
1013 case H5T_STRING: {
1014 size_t str_size = H5Tget_size(type);
1015 if(H5Tis_variable_str(type)>0) {
1016 throw InternalErr(__FILE__, __LINE__,
1017 "print_attr function doesn't handle variable length string, variable length string should be handled separately.");
1018 }
1019 if (str_size == 0){
1020 throw InternalErr(__FILE__, __LINE__, "H5Tget_size() failed.");
1021 }
1022 BESDEBUG("h5", "=print_attr(): H5T_STRING sm_buf=" << (char *) sm_buf
1023 << " size=" << str_size << endl);
1024 // Not sure why the original code add 1 byte to the buffer, perhaps to keep the c-style? KY 2021-04-12
1025#if 0
1026 //rep.resize(str_size+1);
1027#endif
1028 rep.resize(str_size);
1029 strncpy(rep.data(), (char *) sm_buf, str_size);
1030
1031 //Also should add the NULL term at the end. We just need the data in C++.
1032#if 0
1033 //rep[str_size] = '\0';
1034#endif
1035#if 0
1036 char *buf = nullptr;
1037 // This try/catch block is here to protect the allocation of buf.
1038 try {
1039
1040
1041 buf = new char[str_size + 1];
1042 strncpy(buf.data(), (char *) sm_buf, str_size);
1043 buf[str_size] = '\0';
1044 // Not necessarily allocate 3 more bytes.
1045 rep.resize(str_size+3);
1046 snprintf(rep.data(), str_size + 3, "%s", buf);
1047 rep[str_size + 2] = '\0';
1048 delete[] buf; buf = 0;
1049 }
1050 catch (...) {
1051 if( buf ) delete[] buf;
1052 throw;
1053 }
1054#endif
1055 break;
1056 }
1057
1058 default:
1059 break;
1060 } // switch(H5Tget_class(type))
1061
1062 string rep_str(rep.begin(),rep.end());
1063 return rep_str;
1064}
1065
1066D4AttributeType daptype_strrep_to_dap4_attrtype(const string & s){
1067
1068 if (s == "Byte")
1069 return attr_byte_c;
1070 else if (s == "Int8")
1071 return attr_int8_c;
1072 else if (s == "UInt8") // This may never be used.
1073 return attr_uint8_c;
1074 else if (s == "Int16")
1075 return attr_int16_c;
1076 else if (s == "UInt16")
1077 return attr_uint16_c;
1078 else if (s == "Int32")
1079 return attr_int32_c;
1080 else if (s == "UInt32")
1081 return attr_uint32_c;
1082 else if (s == "Int64")
1083 return attr_int64_c;
1084 else if (s == "UInt64")
1085 return attr_uint64_c;
1086 else if (s == "Float32")
1087 return attr_float32_c;
1088 else if (s == "Float64")
1089 return attr_float64_c;
1090 else if (s == "String")
1091 return attr_str_c;
1092 else if (s == "Url")
1093 return attr_url_c;
1094 else
1095 return attr_null_c;
1096
1097
1098}
1099
1100
1114//static BaseType *Get_bt(const string &vname,
1115BaseType *Get_bt(const string &vname,
1116 const string &vpath,
1117 const string &dataset,
1118 hid_t datatype, bool is_dap4)
1119{
1120 BaseType *btp = nullptr;
1121
1122 try {
1123
1124 BESDEBUG("h5", ">Get_bt varname=" << vname << " datatype=" << datatype
1125 << endl);
1126
1127 size_t size = 0;
1128 H5T_sign_t sign = H5T_SGN_ERROR;
1129 switch (H5Tget_class(datatype)) {
1130
1131 case H5T_INTEGER:
1132 {
1133 size = H5Tget_size(datatype);
1134 sign = H5Tget_sign(datatype);
1135 BESDEBUG("h5", "=Get_bt() H5T_INTEGER size = " << size << " sign = "
1136 << sign << endl);
1137
1138 if (sign == H5T_SGN_ERROR) {
1139 throw InternalErr(__FILE__, __LINE__, "cannot retrieve the sign type of the integer");
1140 }
1141 if (size == 0) {
1142 throw InternalErr(__FILE__, __LINE__, "cannot return the size of the datatype");
1143 }
1144 else if (size == 1) { // Either signed char or unsigned char
1145 // DAP2 doesn't support signed char, it maps to DAP int16.
1146 if (sign == H5T_SGN_2) {
1147 if (false == is_dap4) // signed char to DAP2 int16
1148 btp = new HDF5Int16(vname, vpath, dataset);
1149 else
1150 btp = new HDF5Int8(vname,vpath,dataset);
1151 }
1152 else
1153 btp = new HDF5Byte(vname, vpath,dataset);
1154 }
1155 else if (size == 2) {
1156 if (sign == H5T_SGN_2)
1157 btp = new HDF5Int16(vname, vpath,dataset);
1158 else
1159 btp = new HDF5UInt16(vname,vpath, dataset);
1160 }
1161 else if (size == 4) {
1162 if (sign == H5T_SGN_2){
1163 btp = new HDF5Int32(vname, vpath,dataset);
1164 }
1165 else
1166 btp = new HDF5UInt32(vname,vpath, dataset);
1167 }
1168 else if (size == 8) {
1169 if(true == is_dap4) {
1170 if(sign == H5T_SGN_2)
1171 btp = new HDF5Int64(vname,vpath, dataset);
1172 else
1173 btp = new HDF5UInt64(vname,vpath, dataset);
1174 }
1175 else {
1176 throw
1177 InternalErr(__FILE__, __LINE__,
1178 string("Unsupported HDF5 64-bit Integer type:")
1179 + vname);
1180 }
1181 }
1182 }
1183 break;
1184
1185 case H5T_FLOAT:
1186 {
1187 size = H5Tget_size(datatype);
1188 BESDEBUG("h5", "=Get_bt() H5T_FLOAT size = " << size << endl);
1189
1190 if (size == 0) {
1191 throw InternalErr(__FILE__, __LINE__, "cannot return the size of the datatype");
1192 }
1193 else if (size == 4) {
1194 btp = new HDF5Float32(vname,vpath, dataset);
1195 }
1196 else if (size == 8) {
1197 btp = new HDF5Float64(vname,vpath, dataset);
1198 }
1199 }
1200 break;
1201
1202 case H5T_STRING:
1203 btp = new HDF5Str(vname, vpath,dataset);
1204 break;
1205
1206 // The array datatype is rarely,rarely used. So this
1207 // part of code is not reviewed.
1208#if 0
1209 case H5T_ARRAY: {
1210 BaseType *ar_bt = 0;
1211 try {
1212 BESDEBUG("h5",
1213 "=Get_bt() H5T_ARRAY datatype = " << datatype
1214 << endl);
1215
1216 // Get the base datatype of the array
1217 hid_t dtype_base = H5Tget_super(datatype);
1218 ar_bt = Get_bt(vname, dataset, dtype_base);
1219 btp = new HDF5Array(vname, dataset, ar_bt);
1220 delete ar_bt; ar_bt = 0;
1221
1222 // Set the size of the array.
1223 int ndim = H5Tget_array_ndims(datatype);
1224 size = H5Tget_size(datatype);
1225 int nelement = 1;
1226
1227 if (dtype_base < 0) {
1228 throw InternalErr(__FILE__, __LINE__, "cannot return the base datatype");
1229 }
1230 if (ndim < 0) {
1231 throw InternalErr(__FILE__, __LINE__, "cannot return the rank of the array datatype");
1232 }
1233 if (size == 0) {
1234 throw InternalErr(__FILE__, __LINE__, "cannot return the size of the datatype");
1235 }
1236 BESDEBUG(cerr
1237 << "=Get_bt()" << " Dim = " << ndim
1238 << " Size = " << size
1239 << endl);
1240
1241 hsize_t size2[DODS_MAX_RANK];
1242 if(H5Tget_array_dims(datatype, size2) < 0){
1243 throw
1244 InternalErr(__FILE__, __LINE__,
1245 string("Could not get array dims for: ")
1246 + vname);
1247 }
1248
1249
1250 HDF5Array &h5_ar = static_cast < HDF5Array & >(*btp);
1251 for (int dim_index = 0; dim_index < ndim; dim_index++) {
1252 h5_ar.append_dim(size2[dim_index]);
1253 BESDEBUG("h5", "=Get_bt() " << size2[dim_index] << endl);
1254 nelement = nelement * size2[dim_index];
1255 }
1256
1257 h5_ar.set_did(dt_inst.dset);
1258 // Assign the array datatype id.
1259 h5_ar.set_tid(datatype);
1260 h5_ar.set_memneed(size);
1261 h5_ar.set_numdim(ndim);
1262 h5_ar.set_numelm(nelement);
1263 h5_ar.set_length(nelement);
1264 h5_ar.d_type = H5Tget_class(dtype_base);
1265 if (h5_ar.d_type == H5T_NO_CLASS){
1266 throw InternalErr(__FILE__, __LINE__, "cannot return the datatype class identifier");
1267 }
1268 }
1269 catch (...) {
1270 if( ar_bt ) delete ar_bt;
1271 if( btp ) delete btp;
1272 throw;
1273 }
1274 break;
1275 }
1276#endif
1277
1278 // Reference map to DAP URL, check the technical note.
1279 case H5T_REFERENCE:
1280 btp = new HDF5Url(vname, vpath,dataset);
1281 break;
1282
1283 default:
1284 throw InternalErr(__FILE__, __LINE__,
1285 string("Unsupported HDF5 type: ") + vname);
1286 }
1287 }
1288 catch (...) {
1289 if( btp ) delete btp;
1290 throw;
1291 }
1292
1293 if (!btp)
1294 throw InternalErr(__FILE__, __LINE__,
1295 string("Could not make a DAP variable for: ")
1296 + vname);
1297
1298 BESDEBUG("h5", "<Get_bt()" << endl);
1299 return btp;
1300}
1301
1302
1319//static Structure *Get_structure(const string &varname,
1320Structure *Get_structure(const string &varname,const string &vpath,
1321 const string &dataset,
1322 hid_t datatype,bool is_dap4)
1323{
1324 HDF5Structure *structure_ptr = nullptr;
1325 char* memb_name = nullptr;
1326 hid_t memb_type = -1;
1327
1328 BESDEBUG("h5", ">Get_structure()" << datatype << endl);
1329
1330 if (H5Tget_class(datatype) != H5T_COMPOUND)
1331 throw InternalErr(__FILE__, __LINE__,
1332 string("Compound-to-structure mapping error for ")
1333 + varname);
1334
1335 try {
1336 structure_ptr = new HDF5Structure(varname, vpath, dataset);
1337
1338 // Retrieve member types
1339 int nmembs = H5Tget_nmembers(datatype);
1340 BESDEBUG("h5", "=Get_structure() has " << nmembs << endl);
1341 if (nmembs < 0){
1342 throw InternalErr(__FILE__, __LINE__, "cannot retrieve the number of elements");
1343 }
1344 for (int i = 0; i < nmembs; i++) {
1345 memb_name = H5Tget_member_name(datatype, i);
1346 H5T_class_t memb_cls = H5Tget_member_class(datatype, i);
1347 memb_type = H5Tget_member_type(datatype, i);
1348 if (memb_name == nullptr){
1349 throw InternalErr(__FILE__, __LINE__, "cannot retrieve the name of the member");
1350 }
1351 if ((memb_cls < 0) || (memb_type < 0)) {
1352 throw InternalErr(__FILE__, __LINE__,
1353 string("Type mapping error for ")
1354 + string(memb_name) );
1355 }
1356
1357 if (memb_cls == H5T_COMPOUND) {
1358 Structure *s = Get_structure(memb_name, memb_name, dataset, memb_type,is_dap4);
1359 structure_ptr->add_var(s);
1360 delete s; s = 0;
1361 }
1362 else if(memb_cls == H5T_ARRAY) {
1363
1364 BaseType *ar_bt = 0;
1365 BaseType *btp = 0;
1366 Structure *s = 0;
1367 hid_t dtype_base = 0;
1368
1369 try {
1370
1371 // Get the base memb_type of the array
1372 dtype_base = H5Tget_super(memb_type);
1373
1374 // Set the size of the array.
1375 int ndim = H5Tget_array_ndims(memb_type);
1376 size_t size = H5Tget_size(memb_type);
1377 int nelement = 1;
1378
1379 if (dtype_base < 0) {
1380 throw InternalErr(__FILE__, __LINE__, "cannot return the base memb_type");
1381 }
1382 if (ndim < 0) {
1383 throw InternalErr(__FILE__, __LINE__, "cannot return the rank of the array memb_type");
1384 }
1385 if (size == 0) {
1386 throw InternalErr(__FILE__, __LINE__, "cannot return the size of the memb_type");
1387 }
1388
1389 hsize_t size2[DODS_MAX_RANK];
1390 if(H5Tget_array_dims(memb_type, size2) < 0){
1391 throw
1392 InternalErr(__FILE__, __LINE__,
1393 string("Could not get array dims for: ")
1394 + string(memb_name));
1395 }
1396
1397 H5T_class_t array_memb_cls = H5Tget_class(dtype_base);
1398 if(array_memb_cls == H5T_NO_CLASS) {
1399 throw InternalErr(__FILE__, __LINE__,
1400 string("cannot get the correct class for compound type member")
1401 + string(memb_name));
1402 }
1403 if(H5T_COMPOUND == array_memb_cls) {
1404
1405 s = Get_structure(memb_name, memb_name,dataset, dtype_base,is_dap4);
1406 HDF5Array *h5_ar = new HDF5Array(memb_name, dataset, s);
1407
1408 for (int dim_index = 0; dim_index < ndim; dim_index++) {
1409 h5_ar->append_dim(size2[dim_index]);
1410 nelement = nelement * size2[dim_index];
1411 }
1412
1413 h5_ar->set_memneed(size);
1414 h5_ar->set_numdim(ndim);
1415 h5_ar->set_numelm(nelement);
1416 h5_ar->set_length(nelement);
1417
1418 structure_ptr->add_var(h5_ar);
1419 delete h5_ar;
1420
1421 }
1422 else if (H5T_INTEGER == array_memb_cls || H5T_FLOAT == array_memb_cls || H5T_STRING == array_memb_cls) {
1423 ar_bt = Get_bt(memb_name, memb_name,dataset, dtype_base,is_dap4);
1424 HDF5Array *h5_ar = new HDF5Array(memb_name,dataset,ar_bt);
1425
1426 for (int dim_index = 0; dim_index < ndim; dim_index++) {
1427 h5_ar->append_dim(size2[dim_index]);
1428 nelement = nelement * size2[dim_index];
1429 }
1430
1431 h5_ar->set_memneed(size);
1432 h5_ar->set_numdim(ndim);
1433 h5_ar->set_numelm(nelement);
1434 h5_ar->set_length(nelement);
1435
1436 structure_ptr->add_var(h5_ar);
1437 delete h5_ar;
1438 }
1439 if( ar_bt ) delete ar_bt;
1440 if( btp ) delete btp;
1441 if(s) delete s;
1442 H5Tclose(dtype_base);
1443
1444 }
1445 catch (...) {
1446 if( ar_bt ) delete ar_bt;
1447 if( btp ) delete btp;
1448 if(s) delete s;
1449 H5Tclose(dtype_base);
1450 throw;
1451 }
1452
1453 }
1454 else if (memb_cls == H5T_INTEGER || memb_cls == H5T_FLOAT || memb_cls == H5T_STRING) {
1455 BaseType *bt = Get_bt(memb_name, memb_name,dataset, memb_type,is_dap4);
1456 structure_ptr->add_var(bt);
1457 delete bt; bt = 0;
1458 }
1459 else {
1460 free(memb_name);
1461 memb_name = nullptr;
1462 throw InternalErr(__FILE__, __LINE__, "unsupported field datatype inside a compound datatype");
1463 }
1464 // Caller needs to free the memory allocated by the library for memb_name.
1465 if(memb_name != nullptr)
1466 free(memb_name);
1467 }
1468 }
1469 catch (...) {
1470 if( structure_ptr )
1471 delete structure_ptr;
1472 if(memb_name!= nullptr)
1473 free(memb_name);
1474 if(memb_type != -1)
1475 H5Tclose(memb_type);
1476 throw;
1477 }
1478
1479 BESDEBUG("h5", "<Get_structure()" << endl);
1480
1481 return structure_ptr;
1482}
1483
1500
1501// Function to use H5OVISIT to check if dimension scale attributes exist.
1502bool check_dimscale(hid_t fileid) {
1503
1504 bool ret_value = false;
1505 herr_t ret_o= H5OVISIT(fileid, H5_INDEX_NAME, H5_ITER_INC, visit_obj_cb, nullptr);
1506 if(ret_o < 0)
1507 throw InternalErr(__FILE__, __LINE__, "H5OVISIT fails");
1508 else
1509 ret_value =(ret_o >0)?true:false;
1510
1511 return ret_value;
1512}
1513
1514static int
1515visit_obj_cb(hid_t group_id, const char *name, const H5O_info_t *oinfo,
1516 void *_op_data)
1517{
1518
1519 int ret_value = 0;
1520 if(oinfo->type == H5O_TYPE_DATASET) {
1521
1522 hid_t dataset = -1;
1523 dataset = H5Dopen2(group_id,name,H5P_DEFAULT);
1524 if(dataset <0)
1525 throw InternalErr(__FILE__, __LINE__, "H5Dopen2 fails in the H5OVISIT call back function.");
1526
1527 hid_t dspace = -1;
1528 dspace = H5Dget_space(dataset);
1529 if(dspace <0) {
1530 H5Dclose(dataset);
1531 throw InternalErr(__FILE__, __LINE__, "H5Dget_space fails in the H5OVISIT call back function.");
1532 }
1533
1534 // We only support netCDF-4 like dimension scales, that is the dimension scale dataset is 1-D dimension.
1535 if(H5Sget_simple_extent_ndims(dspace) == 1) {
1536 try {
1537 if(true == has_dimscale_attr(dataset))
1538 ret_value = 1;
1539 }
1540 catch(...) {
1541 H5Sclose(dspace);
1542 H5Dclose(dataset);
1543 throw;
1544 }
1545
1546#if 0
1547 //vector<bool>dim_attr_mark;
1548 //dim_attr_mark.resize(4);
1549 //bool dim_attr_mark[4];
1550 int dim_attr_mark[4];
1551 for(int i =0;i<4;i++)
1552 dim_attr_mark[i] = 0;
1553 //int count = 0;
1554 // Check if having "class = DIMENSION_SCALE" and REFERENCE_LIST attributes.
1555 //herr_t ret = H5Aiterate2(dataset, H5_INDEX_NAME, H5_ITER_INC, nullptr, attr_info, &count);
1556 //herr_t ret = H5Aiterate2(dataset, H5_INDEX_NAME, H5_ITER_INC, nullptr, attr_info, dim_attr_mark.data());
1557 herr_t ret = H5Aiterate2(dataset, H5_INDEX_NAME, H5_ITER_INC, nullptr, attr_info, dim_attr_mark);
1558 if(ret < 0) {
1559 H5Sclose(dspace);
1560 H5Dclose(dataset);
1561 throw InternalErr(__FILE__, __LINE__, "H5Aiterate2 fails in the H5OVISIT call back function.");
1562 }
1563
1564 BESDEBUG("h5", "<dset name is " << name <<endl);
1565 //BESDEBUG("h5", "<count is " << count <<endl);
1566 // Find it.
1567 if (dim_attr_mark[0] && dim_attr_mark[1]){
1568 //if (2==count || 3==count) {
1569 if(dspace != -1)
1570 H5Sclose(dspace);
1571 if(dataset != -1)
1572 H5Dclose(dataset);
1573 return 1;
1574 //}
1575 }
1576#endif
1577 }
1578 if(dspace != -1)
1579 H5Sclose(dspace);
1580 if(dataset != -1)
1581 H5Dclose(dataset);
1582 }
1583 return ret_value;
1584}
1585
1586bool has_dimscale_attr(hid_t dataset) {
1587
1588 bool ret_value = false;
1589 string dimscale_attr_name="CLASS";
1590 string dimscale_attr_value="DIMENSION_SCALE";
1591 htri_t dimscale_attr_exist = H5Aexists_by_name(dataset,".",dimscale_attr_name.c_str(),H5P_DEFAULT);
1592 if(dimscale_attr_exist <0)
1593 throw InternalErr(__FILE__, __LINE__, "H5Aexists_by_name fails when checking the CLASS attribute.");
1594 else if(dimscale_attr_exist > 0) {//Attribute CLASS exists
1595
1596 hid_t attr_id = -1;
1597 hid_t atype_id = -1;
1598
1599 // Open the attribute
1600 attr_id = H5Aopen(dataset,dimscale_attr_name.c_str(), H5P_DEFAULT);
1601 if(attr_id < 0)
1602 throw InternalErr(__FILE__, __LINE__, "H5Aopen fails in the attr_info call back function.");
1603
1604 // Get attribute datatype and dataspace
1605 atype_id = H5Aget_type(attr_id);
1606 if(atype_id < 0) {
1607 H5Aclose(attr_id);
1608 throw InternalErr(__FILE__, __LINE__, "H5Aget_type fails in the attr_info call back function.");
1609 }
1610
1611 try {
1612 // Check if finding the attribute CLASS and the value is "DIMENSION_SCALE".
1613 if (H5T_STRING == H5Tget_class(atype_id))
1614 ret_value = check_str_attr_value(attr_id,atype_id,dimscale_attr_value,false);
1615 }
1616 catch(...) {
1617 if(atype_id != -1)
1618 H5Tclose(atype_id);
1619 if(attr_id != -1)
1620 H5Aclose(attr_id);
1621 throw;
1622 }
1623 // Close IDs.
1624 if(atype_id != -1)
1625 H5Tclose(atype_id);
1626 if(attr_id != -1)
1627 H5Aclose(attr_id);
1628 }
1629 return ret_value;
1630
1631}
1632#if 0
1645
1646static herr_t
1647attr_info_dimscale(hid_t loc_id, const char *name, const H5A_info_t *ainfo, void *opdata)
1648{
1649 int *countp = (int*)opdata;
1650
1651 hid_t attr_id = -1;
1652 hid_t atype_id = -1;
1653
1654 // Open the attribute
1655 attr_id = H5Aopen(loc_id, name, H5P_DEFAULT);
1656 if(attr_id < 0)
1657 throw InternalErr(__FILE__, __LINE__, "H5Aopen fails in the attr_info call back function.");
1658
1659 // Get attribute datatype and dataspace
1660 atype_id = H5Aget_type(attr_id);
1661 if(atype_id < 0) {
1662 H5Aclose(attr_id);
1663 throw InternalErr(__FILE__, __LINE__, "H5Aget_type fails in the attr_info call back function.");
1664 }
1665
1666 try {
1667
1668#if 0
1669 // If finding the "REFERENCE_LIST", increases the countp.
1670 if ((H5T_COMPOUND == H5Tget_class(atype_id)) && (strcmp(name,"REFERENCE_LIST")==0)) {
1671 //(*countp)++;
1672 *dimattr_p = 1;
1673 //*dimattr_p = true;
1674 }
1675#endif
1676 // Check if finding the attribute CLASS and the value is "DIMENSION_SCALE".
1677 if (H5T_STRING == H5Tget_class(atype_id)) {
1678 if (strcmp(name,"CLASS") == 0) {
1679 string dim_scale_mark = "DIMENSION_SCALE";
1680 bool is_dim_scale = check_str_attr_value(attr_id,atype_id,dim_scale_mark,false);
1681 if(true == is_dim_scale)
1682 (*countp)++;
1683 }
1684 }
1685 }
1686 catch(...) {
1687 if(atype_id != -1)
1688 H5Tclose(atype_id);
1689 if(attr_id != -1)
1690 H5Aclose(attr_id);
1691 throw;
1692 }
1693
1694 // Close IDs.
1695 if(atype_id != -1)
1696 H5Tclose(atype_id);
1697 if(attr_id != -1)
1698 H5Aclose(attr_id);
1699
1700 return 0;
1701}
1702
1703#endif
1716
1717static herr_t
1718attr_info_dimscale(hid_t loc_id, const char *name, const H5A_info_t *ainfo, void *opdata)
1719{
1720 //bool *countp = (bool*)opdata;
1721 //bool *dimattr_p = (bool*)opdata;
1722 int *dimattr_p = (int*)opdata;
1723
1724#if 0
1725 bool has_reference_list = false;
1726 bool has_dimscale = false;
1727 bool has_name_as_var = false;
1728 bool has_name_as_nc4_purdim = false;
1729#endif
1730
1731
1732 hid_t attr_id = -1;
1733 hid_t atype_id = -1;
1734
1735 // Open the attribute
1736 attr_id = H5Aopen(loc_id, name, H5P_DEFAULT);
1737 if(attr_id < 0)
1738 throw InternalErr(__FILE__, __LINE__, "H5Aopen fails in the attr_info call back function.");
1739
1740 // Get attribute datatype and dataspace
1741 atype_id = H5Aget_type(attr_id);
1742 if(atype_id < 0) {
1743 H5Aclose(attr_id);
1744 throw InternalErr(__FILE__, __LINE__, "H5Aget_type fails in the attr_info call back function.");
1745 }
1746
1747 try {
1748
1749//#if 0
1750 // If finding the "REFERENCE_LIST", increases the countp.
1751 if ((H5T_COMPOUND == H5Tget_class(atype_id)) && (strcmp(name,"REFERENCE_LIST")==0)) {
1752 //(*countp)++;
1753 *dimattr_p = 1;
1754 //*dimattr_p = true;
1755 }
1756//#endif
1757 // Check if finding the CLASS attribute.
1758 if (H5T_STRING == H5Tget_class(atype_id)) {
1759 if (strcmp(name,"NAME") == 0) {
1760
1761 string pure_dimname_mark = "This is a netCDF dimension but not a netCDF variable";
1762 bool is_pure_dim = check_str_attr_value(attr_id,atype_id,pure_dimname_mark,true);
1763
1764 BESDEBUG("h5","pure dimension name yes" << is_pure_dim <<endl);
1765 if(true == is_pure_dim)
1766 *(dimattr_p+1) =1;
1767 //*(dimattr_p+2) =true;
1768 else {
1769 // netCDF save the variable name in the "NAME" attribute.
1770 // We need to retrieve the variable name first.
1771 ssize_t objnamelen = -1;
1772 if ((objnamelen= H5Iget_name(loc_id,nullptr,0))<=0) {
1773 string msg = "Cannot obtain the variable name length." ;
1774 throw InternalErr(__FILE__,__LINE__,msg);
1775 }
1776 vector<char> objname;
1777 objname.resize(objnamelen+1);
1778 if ((objnamelen= H5Iget_name(loc_id,objname.data(),objnamelen+1))<=0) {
1779 string msg = "Cannot obtain the variable name." ;
1780 throw InternalErr(__FILE__,__LINE__,msg);
1781 }
1782
1783 string objname_str = string(objname.begin(),objname.end());
1784
1785 // Must trim the string delimter.
1786 objname_str = objname_str.substr(0,objnamelen);
1787 // Remove the path
1788 string normal_dimname_mark = objname_str.substr(objname_str.find_last_of("/")+1);
1789 bool is_normal_dim = check_str_attr_value(attr_id,atype_id,normal_dimname_mark,false);
1790 if(true == is_normal_dim)
1791 *(dimattr_p+2) = 1;
1792 //*(dimattr_p+3) = true;
1793 }
1794 }
1795 }
1796
1797#if 0
1798 H5T_str_t str_pad = H5Tget_strpad(atype_id);
1799
1800 hid_t aspace_id = -1;
1801 aspace_id = H5Aget_space(attr_id);
1802 if(aspace_id < 0)
1803 throw InternalErr(__FILE__, __LINE__, "H5Aget_space fails in the attr_info call back function.");
1804
1805 // CLASS is a variable-length string
1806 int ndims = H5Sget_simple_extent_ndims(aspace_id);
1807 hsize_t nelmts = 1;
1808
1809 // if it is a scalar attribute, just define number of elements to be 1.
1810 if (ndims != 0) {
1811
1812 vector<hsize_t> asize;
1813 vector<hsize_t> maxsize;
1814 asize.resize(ndims);
1815 maxsize.resize(ndims);
1816
1817 // DAP applications don't care about the unlimited dimensions
1818 // since the applications only care about retrieving the data.
1819 // So we don't check the maxsize to see if it is the unlimited dimension
1820 // attribute.
1821 if (H5Sget_simple_extent_dims(aspace_id, asize.data(), maxsize.data())<0) {
1822 H5Sclose(aspace_id);
1823 throw InternalErr(__FILE__, __LINE__, "Cannot obtain the dim. info in the H5Aiterate2 call back function.");
1824 }
1825
1826 // Return ndims and size[ndims].
1827 for (int j = 0; j < ndims; j++)
1828 nelmts *= asize[j];
1829 } // if(ndims != 0)
1830
1831 size_t ty_size = H5Tget_size(atype_id);
1832 if (0 == ty_size) {
1833 H5Sclose(aspace_id);
1834 throw InternalErr(__FILE__, __LINE__, "Cannot obtain the type size in the H5Aiterate2 call back function.");
1835 }
1836
1837 size_t total_bytes = nelmts * ty_size;
1838 string total_vstring ="";
1839 if(H5Tis_variable_str(atype_id) > 0) {
1840
1841 // Variable length string attribute values only store pointers of the actual string value.
1842 vector<char> temp_buf;
1843 temp_buf.resize(total_bytes);
1844
1845 if (H5Aread(attr_id, atype_id, temp_buf.data()) < 0){
1846 H5Sclose(aspace_id);
1847 throw InternalErr(__FILE__,__LINE__,"Cannot read the attribute in the H5Aiterate2 call back function");
1848 }
1849
1850 char *temp_bp = nullptr;
1851 temp_bp = temp_buf.data();
1852 char* onestring = nullptr;
1853
1854 for (unsigned int temp_i = 0; temp_i <nelmts; temp_i++) {
1855
1856 // This line will assure that we get the real variable length string value.
1857 onestring =*(char **)temp_bp;
1858
1859 if(onestring!= nullptr)
1860 total_vstring +=string(onestring);
1861
1862 // going to the next value.
1863 temp_bp +=ty_size;
1864 }
1865
1866 if ((temp_buf.data()) != nullptr) {
1867 // Reclaim any VL memory if necessary.
1868 if (H5Dvlen_reclaim(atype_id,aspace_id,H5P_DEFAULT,temp_buf.data()) < 0) {
1869 H5Sclose(aspace_id);
1870 throw InternalErr(__FILE__,__LINE__,"Cannot reclaim VL memory in the H5Aiterate2 call back function.");
1871 }
1872 }
1873
1874 }
1875 else {// Fixed-size string, need to retrieve the string value.
1876
1877 // string attribute values
1878 vector<char> temp_buf;
1879 temp_buf.resize(total_bytes);
1880 if (H5Aread(attr_id, atype_id, temp_buf.data()) < 0){
1881 H5Sclose(aspace_id);
1882 throw InternalErr(__FILE__,__LINE__,"Cannot read the attribute in the H5Aiterate2 call back function");
1883 }
1884 string temp_buf_string(temp_buf.begin(),temp_buf.end());
1885 total_vstring = temp_buf_string.substr(0,total_bytes);
1886
1887 // Note: we need to remove the string pad or term to find DIMENSION_SCALE.
1888 if(str_pad != H5T_STR_ERROR)
1889 total_vstring = total_vstring.substr(0,total_vstring.size()-1);
1890 }
1891
1892 // Close attribute data space ID.
1893 if(aspace_id != -1)
1894 H5Sclose(aspace_id);
1895 if(total_vstring == "DIMENSION_SCALE"){
1896 (*countp)++;
1897 }
1898#endif
1899
1900 }
1901 catch(...) {
1902 if(atype_id != -1)
1903 H5Tclose(atype_id);
1904 if(attr_id != -1)
1905 H5Aclose(attr_id);
1906 throw;
1907 }
1908
1909 // Close IDs.
1910 if(atype_id != -1)
1911 H5Tclose(atype_id);
1912 if(attr_id != -1)
1913 H5Aclose(attr_id);
1914
1915 return 0;
1916}
1917
1918
1926
1928void obtain_dimnames(const hid_t file_id,hid_t dset,int ndims, DS_t *dt_inst_ptr,vector<link_info_t> & hdf5_hls) {
1929
1930 htri_t has_dimension_list = -1;
1931
1932 string dimlist_name = "DIMENSION_LIST";
1933 has_dimension_list = H5Aexists(dset,dimlist_name.c_str());
1934
1935 if(has_dimension_list > 0 && ndims > 0) {
1936
1937 hobj_ref_t rbuf;
1938 vector<hvl_t> vlbuf;
1939 vlbuf.resize(ndims);
1940
1941 hid_t attr_id = -1;
1942 hid_t atype_id = -1;
1943 hid_t amemtype_id = -1;
1944 hid_t aspace_id = -1;
1945 hid_t ref_dset = -1;
1946
1947 try {
1948 attr_id = H5Aopen(dset,dimlist_name.c_str(),H5P_DEFAULT);
1949 if (attr_id <0 ) {
1950 string msg = "Cannot open the attribute " + dimlist_name + " of HDF5 dataset "+ string(dt_inst_ptr->name);
1951 throw InternalErr(__FILE__, __LINE__, msg);
1952 }
1953
1954 atype_id = H5Aget_type(attr_id);
1955 if (atype_id <0) {
1956 string msg = "Cannot get the datatype of the attribute " + dimlist_name + " of HDF5 dataset "+ string(dt_inst_ptr->name);
1957 throw InternalErr(__FILE__, __LINE__, msg);
1958 }
1959
1960 amemtype_id = H5Tget_native_type(atype_id, H5T_DIR_ASCEND);
1961 if (amemtype_id < 0) {
1962 string msg = "Cannot get the memory datatype of the attribute " + dimlist_name + " of HDF5 dataset "+ string(dt_inst_ptr->name);
1963 throw InternalErr(__FILE__, __LINE__, msg);
1964
1965 }
1966
1967 if (H5Aread(attr_id,amemtype_id,vlbuf.data()) <0) {
1968 string msg = "Cannot obtain the referenced object for the variable " + string(dt_inst_ptr->name);
1969 throw InternalErr(__FILE__, __LINE__, msg);
1970 }
1971
1972 vector<char> objname;
1973
1974 // The dimension names of variables will be the HDF5 dataset names dereferenced from the DIMENSION_LIST attribute.
1975 for (int i = 0; i < ndims; i++) {
1976
1977 if(vlbuf[i].p == nullptr) {
1978 stringstream sindex ;
1979 sindex <<i;
1980 string msg = "For variable " + string(dt_inst_ptr->name) + "; ";
1981 msg = msg + "the dimension of which the index is "+ sindex.str() + " doesn't exist. ";
1982 throw InternalErr(__FILE__, __LINE__, msg);
1983 }
1984
1985 rbuf =((hobj_ref_t*)vlbuf[i].p)[0];
1986
1987 if ((ref_dset = H5RDEREFERENCE(attr_id, H5R_OBJECT, &rbuf)) < 0) {
1988 string msg = "Cannot dereference from the DIMENSION_LIST attribute for the variable " + string(dt_inst_ptr->name);
1989 throw InternalErr(__FILE__, __LINE__, msg);
1990 }
1991
1992 ssize_t objnamelen = -1;
1993 if ((objnamelen= H5Iget_name(ref_dset,nullptr,0))<=0) {
1994 string msg = "Cannot obtain the dimension name length for the variable " + string(dt_inst_ptr->name);
1995 throw InternalErr(__FILE__,__LINE__,msg);
1996 }
1997
1998 objname.resize(objnamelen+1);
1999 if ((objnamelen= H5Iget_name(ref_dset,objname.data(),objnamelen+1))<=0) {
2000 H5Dclose(ref_dset);
2001 string msg = "Cannot obtain the dimension name for the variable " + string(dt_inst_ptr->name);
2002 throw InternalErr(__FILE__,__LINE__,msg);
2003 }
2004
2005 auto objname_str = string(objname.begin(),objname.end());
2006
2007 // Must trim the string delimter.
2008 string trim_objname = objname_str.substr(0,objnamelen);
2009
2010 // We need to check if there are hardlinks for this variable.
2011 // If yes, we need to find the hardlink that has the shortest path and at the ancestor group
2012 // of all links.
2013 H5O_info_t obj_info;
2014 if(H5OGET_INFO(ref_dset,&obj_info)<0) {
2015 H5Dclose(ref_dset);
2016 string msg = "Cannot obtain the object info for the dimension variable " + objname_str;
2017 throw InternalErr(__FILE__,__LINE__,msg);
2018 }
2019
2020 // This dimension indeed has hard links.
2021 if(obj_info.rc > 1) {
2022
2023 // 1. Search the hdf5_hls to see if the address is inside
2024 // if yes,
2025 // obtain the hard link which has the shortest path, use this as the dimension name.
2026 // else
2027 // search all the hardlinks with the callback.
2028 // obtain the shortest path, add this to hdf5_hls.
2029
2030 bool link_find = false;
2031
2032 // If finding the object in the hdf5_hls, obtain the hardlink and make it the dimension name(trim_objname).
2033 for (unsigned int i = 0; i <hdf5_hls.size();i++) {
2034#if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
2035 int token_cmp = -1;
2036 if(H5Otoken_cmp(ref_dset,&(obj_info.token),&(hdf5_hls[i].link_addr),&token_cmp) <0)
2037 throw InternalErr(__FILE__,__LINE__,"H5Otoken_cmp failed");
2038 if(!token_cmp) {
2039#else
2040 if(obj_info.addr == hdf5_hls[i].link_addr) {
2041#endif
2042 trim_objname = '/'+hdf5_hls[i].slink_path;
2043 link_find = true;
2044 break;
2045 }
2046 }
2047
2048 // The hard link is not in the hdf5_hls, need to iterate all objects and find those hardlinks.
2049 if(link_find == false) {
2050
2051#if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
2052 typedef struct {
2053 unsigned link_unvisited;
2054 H5O_token_t link_addr;
2055 vector<string> hl_names;
2056 } t_link_info_t;
2057#else
2058 typedef struct {
2059 unsigned link_unvisited;
2060 haddr_t link_addr;
2061 vector<string> hl_names;
2062 } t_link_info_t;
2063#endif
2064
2065 t_link_info_t t_li_info;
2066 t_li_info.link_unvisited = obj_info.rc;
2067
2068#if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
2069 memcpy(&t_li_info.link_addr,&obj_info.token,sizeof(H5O_token_t));
2070#else
2071 t_li_info.link_addr = obj_info.addr;
2072#endif
2073
2074 if(H5Lvisit(file_id, H5_INDEX_NAME, H5_ITER_NATIVE, visit_link_cb, (void*)&t_li_info) < 0) {
2075 H5Dclose(ref_dset);
2076 string err_msg;
2077 err_msg = "Find all hardlinks: H5Lvisit failed to iterate all the objects";
2078 throw InternalErr(__FILE__,__LINE__,err_msg);
2079 }
2080#if 0
2081for(int i = 0; i<t_li_info.hl_names.size();i++)
2082 cerr<<"hl name is "<<t_li_info.hl_names[i] <<endl;
2083#endif
2084
2085 string shortest_hl = obtain_shortest_ancestor_path(t_li_info.hl_names);
2086//cerr<<"shortest_hl is "<<shortest_hl <<endl;
2087 if(shortest_hl =="") {
2088 H5Dclose(ref_dset);
2089 string err_msg;
2090 err_msg = "The shortest hardlink is not located under an ancestor group of all links.";
2091 err_msg +="This is not supported by netCDF4 data model and the current Hyrax DAP4 implementation.";
2092 throw InternalErr(__FILE__,__LINE__,err_msg);
2093 }
2094
2095 // Save this link that holds the shortest path for future use.
2096 link_info_t new_hdf5_hl;
2097#if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
2098 memcpy(&new_hdf5_hl.link_addr,&obj_info.token,sizeof(H5O_token_t));
2099#else
2100 new_hdf5_hl.link_addr = obj_info.addr;
2101#endif
2102 new_hdf5_hl.slink_path = shortest_hl;
2103 hdf5_hls.push_back(new_hdf5_hl);
2104 trim_objname = '/'+shortest_hl;
2105
2106 }
2107 }
2108 // Need to save the dimension names without the path
2109
2110 dt_inst_ptr->dimnames.push_back(trim_objname.substr(trim_objname.find_last_of("/")+1));
2111 dt_inst_ptr->dimnames_path.push_back(trim_objname);
2112
2113 if(H5Dclose(ref_dset)<0) {
2114 throw InternalErr(__FILE__,__LINE__,"Cannot close the HDF5 dataset in the function obtain_dimnames().");
2115 }
2116 objname.clear();
2117 }// for (vector<Dimension *>::iterator ird is var->dims.begin()
2118 if(vlbuf.empty()== false) {
2119
2120 if ((aspace_id = H5Aget_space(attr_id)) < 0) {
2121 string msg = "Cannot close the HDF5 attribute space successfully for <DIMENSION_LIST> of the variable "+string(dt_inst_ptr->name);
2122 throw InternalErr(__FILE__,__LINE__,msg);
2123 }
2124
2125 if (H5Dvlen_reclaim(amemtype_id,aspace_id,H5P_DEFAULT,(void*)vlbuf.data())<0) {
2126 throw InternalErr(__FILE__,__LINE__,"Cannot reclaim the variable length memory in the function obtain_dimnames()");
2127 }
2128
2129 H5Sclose(aspace_id);
2130
2131 }
2132
2133 H5Tclose(atype_id);
2134 H5Tclose(amemtype_id);
2135 H5Aclose(attr_id);
2136
2137 }
2138
2139 catch(...) {
2140
2141 if(atype_id != -1)
2142 H5Tclose(atype_id);
2143
2144 if(amemtype_id != -1)
2145 H5Tclose(amemtype_id);
2146
2147 if(aspace_id != -1)
2148 H5Sclose(aspace_id);
2149
2150 if(attr_id != -1)
2151 H5Aclose(attr_id);
2152
2153 throw;
2154 }
2155
2156 }
2157 return ;
2158}
2159
2160void write_vlen_str_attrs(hid_t attr_id,hid_t ty_id, const DSattr_t * attr_inst_ptr,D4Attribute *d4_attr, AttrTable* d2_attr,bool is_dap4){
2161
2162 BESDEBUG("h5","attribute name " << attr_inst_ptr->name <<endl);
2163 BESDEBUG("h5","attribute size " <<attr_inst_ptr->need <<endl);
2164 BESDEBUG("h5","attribute type size " <<(int)(H5Tget_size(ty_id))<<endl);
2165
2166 hid_t temp_space_id = H5Aget_space(attr_id);
2167 BESDEBUG("h5","attribute calculated size "<<(int)(H5Tget_size(ty_id)) *(int)(H5Sget_simple_extent_npoints(temp_space_id)) <<endl);
2168 if(temp_space_id <0) {
2169 H5Tclose(ty_id);
2170 H5Aclose(attr_id);
2171 if(d4_attr)
2172 delete d4_attr;
2173 throw InternalErr(__FILE__, __LINE__, "unable to read HDF5 attribute data");
2174 }
2175
2176 vector<char> temp_buf;
2177 // Variable length string attribute values only store pointers of the actual string value.
2178 temp_buf.resize((size_t)attr_inst_ptr->need);
2179
2180 if (H5Aread(attr_id, ty_id, temp_buf.data()) < 0) {
2181 H5Tclose(ty_id);
2182 H5Aclose(attr_id);
2183 H5Sclose(temp_space_id);
2184 if(d4_attr)
2185 delete d4_attr;
2186 throw InternalErr(__FILE__, __LINE__, "unable to read HDF5 attribute data");
2187 }
2188
2189 char *temp_bp;
2190 temp_bp = temp_buf.data();
2191 for (unsigned int temp_i = 0; temp_i <attr_inst_ptr->nelmts; temp_i++) {
2192
2193 // This line will assure that we get the real variable length string value.
2194 char* onestring =*(char **)temp_bp;
2195
2196 // Change the C-style string to C++ STD string just for easy appending the attributes in DAP.
2197 if (onestring !=nullptr) {
2198 string tempstring(onestring);
2199 if(true == is_dap4)
2200 d4_attr->add_value(tempstring);
2201 else
2202 d2_attr->append_attr(attr_inst_ptr->name,"String",tempstring);
2203 }
2204
2205 temp_bp +=H5Tget_size(ty_id);
2206 }
2207 if (temp_buf.empty() != true) {
2208
2209 // Reclaim any VL memory if necessary.
2210 herr_t ret_vlen_claim;
2211 ret_vlen_claim = H5Dvlen_reclaim(ty_id,temp_space_id,H5P_DEFAULT,temp_buf.data());
2212 if(ret_vlen_claim < 0){
2213 H5Tclose(ty_id);
2214 H5Aclose(attr_id);
2215 H5Sclose(temp_space_id);
2216 if(d4_attr)
2217 delete d4_attr;
2218 throw InternalErr(__FILE__, __LINE__, "Cannot reclaim the memory buffer of the HDF5 variable length string.");
2219 }
2220
2221 temp_buf.clear();
2222 }
2223 H5Sclose(temp_space_id);
2224}
2225
2226bool check_str_attr_value(hid_t attr_id,hid_t atype_id,const string & value_to_compare,bool check_substr) {
2227
2228 bool ret_value = false;
2229
2230 H5T_str_t str_pad = H5Tget_strpad(atype_id);
2231 if(str_pad == H5T_STR_ERROR)
2232 throw InternalErr(__FILE__, __LINE__, "Fail to obtain string pad.");
2233
2234 hid_t aspace_id = -1;
2235 aspace_id = H5Aget_space(attr_id);
2236 if(aspace_id < 0)
2237 throw InternalErr(__FILE__, __LINE__, "Fail to obtain attribute space.");
2238
2239 int ndims = H5Sget_simple_extent_ndims(aspace_id);
2240 if(ndims < 0) {
2241 H5Sclose(aspace_id);
2242 throw InternalErr(__FILE__, __LINE__, "Fail to obtain number of dimensions.");
2243 }
2244
2245 hsize_t nelmts = 1;
2246
2247 // if it is a scalar attribute, just define number of elements to be 1.
2248 if (ndims != 0) {
2249
2250 vector<hsize_t> asize;
2251 asize.resize(ndims);
2252 if (H5Sget_simple_extent_dims(aspace_id, asize.data(), nullptr)<0) {
2253 H5Sclose(aspace_id);
2254 throw InternalErr(__FILE__, __LINE__, "Fail to obtain the dimension info.");
2255 }
2256
2257 // Return ndims and size[ndims].
2258 for (int j = 0; j < ndims; j++)
2259 nelmts *= asize[j];
2260 } // if(ndims != 0)
2261
2262 size_t ty_size = H5Tget_size(atype_id);
2263 if (0 == ty_size) {
2264 H5Sclose(aspace_id);
2265 throw InternalErr(__FILE__, __LINE__, "Fail to obtain the type size.");
2266 }
2267
2268 size_t total_bytes = nelmts * ty_size;
2269 string total_vstring ="";
2270 if(H5Tis_variable_str(atype_id) > 0) {
2271
2272 // Variable length string attribute values only store pointers of the actual string value.
2273 vector<char> temp_buf;
2274 temp_buf.resize(total_bytes);
2275
2276 if (H5Aread(attr_id, atype_id, temp_buf.data()) < 0){
2277 H5Sclose(aspace_id);
2278 throw InternalErr(__FILE__,__LINE__,"Fail to read the attribute.");
2279 }
2280
2281 char *temp_bp = nullptr;
2282 temp_bp = temp_buf.data();
2283
2284 for (unsigned int temp_i = 0; temp_i <nelmts; temp_i++) {
2285
2286 // This line will assure that we get the real variable length string value.
2287 char* onestring =*(char **)temp_bp;
2288
2289 if(onestring!= nullptr)
2290 total_vstring +=string(onestring);
2291
2292 // going to the next value.
2293 temp_bp +=ty_size;
2294 }
2295
2296 if ((temp_buf.data()) != nullptr) {
2297 // Reclaim any VL memory if necessary.
2298 if (H5Dvlen_reclaim(atype_id,aspace_id,H5P_DEFAULT,temp_buf.data()) < 0) {
2299 H5Sclose(aspace_id);
2300 throw InternalErr(__FILE__,__LINE__,"Fail to reclaim VL memory.");
2301 }
2302 }
2303
2304 }
2305 else {// Fixed-size string, need to retrieve the string value.
2306
2307 // string attribute values
2308 vector<char> temp_buf;
2309 temp_buf.resize(total_bytes);
2310 if (H5Aread(attr_id, atype_id, temp_buf.data()) < 0){
2311 H5Sclose(aspace_id);
2312 throw InternalErr(__FILE__,__LINE__,"Fail to read the attribute.");
2313 }
2314 string temp_buf_string(temp_buf.begin(),temp_buf.end());
2315 total_vstring = temp_buf_string.substr(0,total_bytes);
2316
2317 // Note: we need to remove the string pad or term to find DIMENSION_SCALE.
2318 if(str_pad != H5T_STR_ERROR)
2319 total_vstring = total_vstring.substr(0,total_vstring.size()-1);
2320 }
2321
2322 // Close attribute data space ID.
2323 if(aspace_id != -1)
2324 H5Sclose(aspace_id);
2325
2326 if(false == check_substr) {
2327 if(total_vstring == value_to_compare)
2328 ret_value = true;
2329 }
2330 else {
2331 if(total_vstring.size()>=value_to_compare.size()) {
2332 if( 0 == total_vstring.compare(0,value_to_compare.size(),value_to_compare))
2333 ret_value = true;
2334 }
2335 }
2336 return ret_value;
2337}
2338
2339// Call back function used by H5Lvisit that iterates all HDF5 objects.
2340static int
2341visit_link_cb(hid_t group_id, const char *name, const H5L_info_t *linfo,
2342 void *_op_data)
2343{
2344#if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
2345 typedef struct {
2346 unsigned link_unvisited;
2347 H5O_token_t link_addr;
2348 vector<string> hl_names;
2349 } t_link_info_t;
2350#else
2351 typedef struct {
2352 unsigned link_unvisited;
2353 haddr_t link_addr;
2354 vector<string> hl_names;
2355 } t_link_info_t;
2356#endif
2357
2358 t_link_info_t *op_data = (t_link_info_t *)_op_data;
2359 int ret = 0;
2360
2361 // We only need the hard link info.
2362 if(linfo->type == H5L_TYPE_HARD) {
2363#if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
2364 int token_cmp = -1;
2365 if(H5Otoken_cmp(group_id,&(op_data->link_addr),&(linfo->u.token),&token_cmp) <0)
2366 throw InternalErr(__FILE__,__LINE__,"H5Otoken_cmp failed");
2367 if(!token_cmp) {
2368#else
2369 if(op_data->link_addr == linfo->u.address) {
2370#endif
2371 op_data->link_unvisited = op_data->link_unvisited -1;
2372 string tmp_str(name,name+strlen(name));
2373 op_data->hl_names.push_back(tmp_str);
2374 // Once visiting all hard links, stop.
2375 if(op_data->link_unvisited == 0)
2376 ret = 1;
2377 }
2378
2379 }
2380 return ret;
2381
2382}
2383
2384// Obtain the shortest path of all hard links of the object.
2385std::string obtain_shortest_ancestor_path(const std::vector<std::string> & hls) {
2386
2387 vector<string> hls_path;
2388 char slash = '/';
2389 bool hl_under_root = false;
2390 string ret_str ="";
2391 unsigned i = 0;
2392
2393 for (i= 0; i<hls.size(); i++) {
2394
2395 size_t path_pos = hls[i].find_last_of(slash);
2396
2397 // The hard link may be under the root group,
2398 // This is the shortest path, we will return this path.
2399 if(path_pos == std::string::npos) {
2400 //Found
2401 hl_under_root = true;
2402 break;
2403 }
2404 else {
2405 string tmp_str = hls[i].substr(0,path_pos+1);
2406 hls_path.push_back(tmp_str);
2407 }
2408 }
2409
2410 if(hl_under_root)
2411 ret_str = hls[i];
2412
2413 else {
2414 // We just need to find the minimum size.
2415 unsigned short_path_index = 0;
2416 unsigned min_path_size = hls_path[0].size();
2417
2418 // Find the shortest path index
2419 for(unsigned j = 1; j <hls_path.size();j++) {
2420 if(min_path_size>hls_path[j].size()) {
2421 min_path_size = hls_path[j].size();
2422 short_path_index = j;
2423 }
2424 }
2425 string tmp_sp = hls_path[short_path_index];
2426 ret_str = hls[short_path_index];
2427
2428 //check if all hardlinks have a common ancestor link
2429 // If not, set the return value be the empty string.
2430 for(unsigned j = 0; j <hls_path.size();j++) {
2431 if(hls_path[j].find(tmp_sp)!=0) {
2432 ret_str ="";
2433 break;
2434 }
2435 }
2436 }
2437 return ret_str;
2438
2439
2440}
A class for handling all types of array in HDF5 for the default option.
This class provides a way to map HDF5 byte to DAP Byte for the default option.
A class for mapping HDF5 32-bit float to DAP for the default option.
A class for mapping HDF5 64-bit float to DAP for the default option.
A class for HDF5 signed 16 bit integer type.
This class provides a way to map HDF5 32 bit integer to DAP Int32 for the default option.
This class provides a way to map HDF5 Int64 to DAP Int64 for the default option.
This class provides a way to map HDF5 int8 to DAP Int8 for the default option.
include the entry functions to execute the handlers
This class that translates HDF5 string into DAP string for the default option.
This class converts HDF5 compound type into DAP structure for the default option.
This class provides a way to map unsigned HDF5 16 bit integer to DAP UInt16 for the default option.
This class provides a way to map unsigned HDF5 32 bit integer to DAP UInt32.
This class provides a way to map HDF5 uint64 to DAP UInt64 for the default option.
This class generates DAP URL type for the default option.
void set_numdim(int ndims)
remembers number of dimensions of this array.
Definition: HDF5Array.cc:1732
void set_numelm(int nelms)
remembers number of elements in this array.
Definition: HDF5Array.cc:1736
void set_memneed(size_t need)
remembers memory size needed.
Definition: HDF5Array.cc:1728
string print_attr(hid_t type, int loc, void *sm_buf)
Definition: h5get.cc:868
bool check_h5str(hid_t h5type)
Definition: h5get.cc:849
hid_t get_fileid(const char *filename)
Definition: h5get.cc:412
string get_dap_type(hid_t type, bool is_dap4)
Definition: h5get.cc:292
void close_fileid(hid_t fid)
Definition: h5get.cc:434
void get_dataset(hid_t pid, const string &dname, DS_t *dt_inst_ptr)
Definition: h5get.cc:453
hid_t get_attr_info(hid_t dset, int index, bool is_dap4, DSattr_t *attr_inst_ptr, bool *ignore_attr_ptr)
Definition: h5get.cc:90
const int DODS_MAX_RANK
Maximum number of dimensions in an array(default option only).
Definition: hdf5_handler.h:63
A structure for DDS generation.
Definition: hdf5_handler.h:71
char name[DODS_NAMELEN]
Name of HDF5 group or dataset.
Definition: hdf5_handler.h:73
A structure for DAS generation.
Definition: hdf5_handler.h:96
char name[DODS_NAMELEN]
Name of HDF5 group or dataset.
Definition: hdf5_handler.h:98
hsize_t nelmts
Number of elements.
Definition: hdf5_handler.h:106
hsize_t need
Memory space needed to hold nelmts type.
Definition: hdf5_handler.h:108