bes Updated for version 3.20.13
HDF5CF.cc
Go to the documentation of this file.
1// This file is part of the hdf5_handler implementing for the CF-compliant
2// Copyright (c) 2011-2016 The HDF Group, Inc. and OPeNDAP, Inc.
3//
4// This is free software; you can redistribute it and/or modify it under the
5// terms of the GNU Lesser General Public License as published by the Free
6// Software Foundation; either version 2.1 of the License, or (at your
7// option) any later version.
8//
9// This software is distributed in the hope that it will be useful, but
10// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
12// License for more details.
13//
14// You should have received a copy of the GNU Lesser General Public
15// License along with this library; if not, write to the Free Software
16// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17//
18// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
19// You can contact The HDF Group, Inc. at 1800 South Oak Street,
20// Suite 203, Champaign, IL 61820
21
36
37#include <sstream>
38#include <algorithm>
39#include <functional>
40#include <climits>
41#include "HDF5CF.h"
42#include "h5cfdaputil.h"
43#include "HDF5RequestHandler.h"
44#include "h5apicompatible.h"
45#include "BESDebug.h"
46
47using namespace HDF5CF;
48
49Var::Var(Var *var)
50{
51
52 newname = var->newname;
53 name = var->name;
54 fullpath = var->fullpath;
55 rank = var->rank;
56 total_elems = var->total_elems;
57 zero_storage_size = var->zero_storage_size;
58 dtype = var->dtype;
59 comp_ratio = var->comp_ratio;
60 unsupported_attr_dtype = var->unsupported_attr_dtype;
61 unsupported_attr_dspace = var->unsupported_attr_dspace;
62 unsupported_dspace = var->unsupported_dspace;
63 unsupported_attr_dspace = var->unsupported_attr_dspace;
64 dimnameflag = var->dimnameflag;
65 coord_attr_add_path = var->coord_attr_add_path;
66
67 for (auto ira = var->attrs.begin(); ira != var->attrs.end(); ++ira) {
68 Attribute* attr = new Attribute();
69 attr->name = (*ira)->name;
70 attr->newname = (*ira)->newname;
71 attr->dtype = (*ira)->dtype;
72 attr->count = (*ira)->count;
73 attr->strsize = (*ira)->strsize;
74 attr->fstrsize = (*ira)->fstrsize;
75 attr->value = (*ira)->value;
76 attrs.push_back(attr);
77 }
78
79 for (auto ird = var->dims.begin(); ird != var->dims.end(); ++ird) {
80 Dimension *dim = new Dimension((*ird)->size);
81 dim->name = (*ird)->name;
82 dim->newname = (*ird)->newname;
83 dim->unlimited_dim = (*ird)->unlimited_dim;
84 dims.push_back(dim);
85 }
86
87}
88
89bool CVar::isLatLon() const
90{
91
92 bool ret_value = false;
93 if (CV_EXIST == this->cvartype || CV_MODIFY == this->cvartype || CV_SPECIAL == this->cvartype) {
94 string attr_name = "units";
95 string lat_unit_value = "degrees_north";
96 string lon_unit_value = "degrees_east";
97
98 for (auto ira = this->attrs.begin(); ira != this->attrs.end(); ira++) {
99
100 if ((H5FSTRING == (*ira)->getType()) || (H5VSTRING == (*ira)->getType())) {
101 if (attr_name == (*ira)->newname) {
102 string attr_value1((*ira)->getValue().begin(), (*ira)->getValue().end());
103
104 if ((*ira)->getCount() == 1) {
105 string attr_value((*ira)->getValue().begin(), (*ira)->getValue().end());
106 if (attr_value.compare(0, lat_unit_value.size(), lat_unit_value) == 0) {
107 if (attr_value.size() == lat_unit_value.size()) {
108 ret_value = true;
109 break;
110 }
111 else if (attr_value.size() == (lat_unit_value.size() + 1)) {
112 if (attr_value[attr_value.size() - 1] == '\0'
113 || attr_value[attr_value.size() - 1] == ' ') {
114 ret_value = true;
115 break;
116 }
117 }
118 }
119 else if (attr_value.compare(0, lon_unit_value.size(), lon_unit_value) == 0) {
120 if (attr_value.size() == lon_unit_value.size()) {
121 ret_value = true;
122 break;
123 }
124 else if (attr_value.size() == (lon_unit_value.size() + 1)) {
125 if (attr_value[attr_value.size() - 1] == '\0'
126 || attr_value[attr_value.size() - 1] == ' ') {
127 ret_value = true;
128 break;
129 }
130 }
131
132 }
133
134 }
135 }
136 }
137 }
138 }
139 else if (this->cvartype == CV_LAT_MISS || this->cvartype == CV_LON_MISS) ret_value = true;
140 return ret_value;
141
142}
143File::~File()
144{
145
146 if (this->fileid >= 0) {
147 if (this->rootid >= 0) {
148 for_each(this->groups.begin(), this->groups.end(), delete_elem());
149 for_each(this->vars.begin(), this->vars.end(), delete_elem());
150 for_each(this->root_attrs.begin(), this->root_attrs.end(), delete_elem());
151 H5Gclose(rootid);
152 }
153 }
154}
155
156Group::~Group()
157{
158 for_each(this->attrs.begin(), this->attrs.end(), delete_elem());
159}
160
161Var::~Var()
162{
163 for_each(this->dims.begin(), this->dims.end(), delete_elem());
164 for_each(this->attrs.begin(), this->attrs.end(), delete_elem());
165}
166
167
168void File::Retrieve_H5_Info(const char * /*path*/, hid_t file_id, bool include_attr)
169{
170
171 BESDEBUG("h5", "coming to Retrieve_H5_Info" <<endl);
172
173 if (true == include_attr) {
174
175 // Obtain the BES key to check the ignored objects
176 // We will only use DAS to output these information.
177 this->check_ignored = HDF5RequestHandler::get_check_ignore_obj();
178 if (true == this->check_ignored) this->add_ignored_info_page_header();
179
180 }
181
182 hid_t root_id;
183 if ((root_id = H5Gopen(file_id, "/", H5P_DEFAULT)) < 0) {
184 throw1("Cannot open the HDF5 root group ");
185 }
186 this->rootid = root_id;
187 try {
188 this->Retrieve_H5_Obj(root_id, "/", include_attr);
189 }
190 catch (...) {
191 throw;
192 }
193
194 // Obtain attributes only necessary
195 if (true == include_attr) {
196
197 // Find the file(root group) attribute
198
199 // Obtain the object type, such as group or dataset.
200 H5O_info_t oinfo;
201 int num_attrs = 0;
202
203 if (H5OGET_INFO(root_id, &oinfo) < 0)
204 throw1("Error obtaining the info for the root group");
205
206 num_attrs = (int)(oinfo.num_attrs);
207 bool temp_unsup_attr_atype = false;
208 bool temp_unsup_attr_dspace = false;
209
210 for (int j = 0; j < num_attrs; j++) {
211 auto attr = new Attribute();
212 try {
213 this->Retrieve_H5_Attr_Info(attr, root_id, j, temp_unsup_attr_atype, temp_unsup_attr_dspace);
214 }
215 catch (...) {
216 delete attr;
217 throw;
218
219 }
220 this->root_attrs.push_back(attr);
221 }
222
223 this->unsupported_attr_dtype = temp_unsup_attr_atype;
224 this->unsupported_attr_dspace = temp_unsup_attr_dspace;
225 }
226}
227
228void File::Retrieve_H5_Obj(hid_t grp_id, const char*gname, bool include_attr)
229{
230
231 // Iterate through the file to see the members of the group from the root.
232 H5G_info_t g_info;
233 hsize_t nelems = 0;
234
235 if (H5Gget_info(grp_id, &g_info) < 0)
236 throw2("Counting hdf5 group elements error for ", gname);
237 nelems = g_info.nlinks;
238
239 ssize_t oname_size = 0;
240 for (hsize_t i = 0; i < nelems; i++) {
241
242 hid_t cgroup = -1;
243 hid_t cdset = -1;
244 Group *group = nullptr;
245 Var *var = nullptr;
246 Attribute *attr = nullptr;
247
248 try {
249
250 size_t dummy_name_len = 1;
251
252 // Query the length of object name.
253 oname_size = H5Lget_name_by_idx(grp_id, ".", H5_INDEX_NAME, H5_ITER_NATIVE, i, nullptr, dummy_name_len,
254 H5P_DEFAULT);
255 if (oname_size <= 0)
256 throw2("Error getting the size of the hdf5 object from the group: ", gname);
257
258 // Obtain the name of the object
259 vector<char> oname;
260 oname.resize((size_t) oname_size + 1);
261
262 if (H5Lget_name_by_idx(grp_id, ".", H5_INDEX_NAME, H5_ITER_NATIVE, i, oname.data(), (size_t) (oname_size + 1),
263 H5P_DEFAULT) < 0)
264 throw2("Error getting the hdf5 object name from the group: ", gname);
265
266 // Check if it is a hard link or a soft link
267 H5L_info_t linfo;
268 if (H5Lget_info(grp_id, oname.data(), &linfo, H5P_DEFAULT) < 0)
269 throw2("HDF5 link name error from ", gname);
270
271 // We ignore soft links and external links for the CF options
272 if (H5L_TYPE_SOFT == linfo.type || H5L_TYPE_EXTERNAL == linfo.type) {
273 if (true == include_attr && true == check_ignored) {
274 this->add_ignored_info_links_header();
275 string full_path_name;
276 string temp_oname(oname.begin(), oname.end());
277 full_path_name = (
278 (string(gname) != "/") ?
279 (string(gname) + "/" + temp_oname.substr(0, temp_oname.size() - 1)) :
280 ("/" + temp_oname.substr(0, temp_oname.size() - 1)));
281 this->add_ignored_info_links(full_path_name);
282
283 }
284 continue;
285 }
286
287 // Obtain the object type, such as group or dataset.
288 H5O_info_t oinfo;
289
290 if (H5OGET_INFO_BY_IDX(grp_id, ".", H5_INDEX_NAME, H5_ITER_NATIVE, i, &oinfo, H5P_DEFAULT) < 0)
291 throw2("Error obtaining the info for the object ", string(oname.begin(), oname.end()));
292
293 H5O_type_t obj_type = oinfo.type;
294
295 switch (obj_type) {
296
297 case H5O_TYPE_GROUP: {
298
299 // Obtain the full path name
300 string full_path_name;
301 string temp_oname(oname.begin(), oname.end());
302
303 full_path_name = (
304 (string(gname) != "/") ?
305 (string(gname) + "/" + temp_oname.substr(0, temp_oname.size() - 1)) :
306 ("/" + temp_oname.substr(0, temp_oname.size() - 1)));
307
308 cgroup = H5Gopen(grp_id, full_path_name.c_str(), H5P_DEFAULT);
309 if (cgroup < 0)
310 throw2("Error opening the group ", full_path_name);
311
312 group = new Group();
313 group->path = full_path_name;
314 group->newname = full_path_name;
315
316 // Retrieve group attribute if the attribute flag is true
317 if (true == include_attr) {
318
319 int num_attrs = (int)(oinfo.num_attrs);
320 bool temp_unsup_attr_dtype = false;
321 bool temp_unsup_attr_dspace = false;
322
323 for (int j = 0; j < num_attrs; j++) {
324
325 attr = new Attribute();
326 Retrieve_H5_Attr_Info(attr, cgroup, j, temp_unsup_attr_dtype, temp_unsup_attr_dspace);
327 group->attrs.push_back(attr);
328 attr = nullptr;
329 }
330
331 group->unsupported_attr_dtype = temp_unsup_attr_dtype;
332 group->unsupported_attr_dspace = temp_unsup_attr_dspace;
333 }
334 this->groups.push_back(group);
335 Retrieve_H5_Obj(cgroup, full_path_name.c_str(), include_attr);
336 if (H5Gclose(cgroup) < 0)
337 throw2("Error closing the group ", full_path_name);
338 }
339 break;
340 case H5O_TYPE_DATASET: {
341
342 // Obtain the absolute path of the HDF5 dataset
343 string temp_oname(oname.begin(), oname.end());
344 string full_path_name = (
345 (string(gname) != "/") ?
346 (string(gname) + "/" + temp_oname.substr(0, temp_oname.size() - 1)) :
347 ("/" + temp_oname.substr(0, temp_oname.size() - 1)));
348
349 var = new Var();
350 var->name = temp_oname.substr(0, temp_oname.size() - 1);
351 var->fullpath = full_path_name;
352
353 // newname is for the final CF name
354 var->newname = full_path_name;
355
356 cdset = H5Dopen(grp_id, full_path_name.c_str(), H5P_DEFAULT);
357 if (cdset < 0)
358 throw2("Error opening the HDF5 dataset ", full_path_name);
359
360 // Retrieve the HDF5 dataset datatype, return the flag for unsupported types.
361 bool temp_unsup_var_dtype = false;
362 Retrieve_H5_VarType(var, cdset, full_path_name, temp_unsup_var_dtype);
363
364 // Update the unsupported datatype flag
365 if (!this->unsupported_var_dtype && temp_unsup_var_dtype) this->unsupported_var_dtype = true;
366
367 // Retrieve the HDF5 dataset data space, return the flag for unsupported dataspaces.
368 bool temp_unsup_var_dspace = false;
369 Retrieve_H5_VarDim(var, cdset, full_path_name, temp_unsup_var_dspace);
370
371 // Update the unsupported data space flag
372 if (!this->unsupported_var_dspace && temp_unsup_var_dspace) this->unsupported_var_dspace = true;
373
374 hsize_t d_storage_size = H5Dget_storage_size(cdset);
375 var->zero_storage_size =(d_storage_size ==0);
376 var->comp_ratio = Retrieve_H5_VarCompRatio(var, cdset);
377
378 // Retrieve the attribute info. if asked
379 if (true == include_attr) {
380
381 int num_attrs = (int)(oinfo.num_attrs);
382 bool temp_unsup_attr_dtype = false;
383 bool temp_unsup_attr_dspace = false;
384
385 for (int j = 0; j < num_attrs; j++) {
386
387 attr = new Attribute();
388
389 Retrieve_H5_Attr_Info(attr, cdset, j, temp_unsup_attr_dtype, temp_unsup_attr_dspace);
390 var->attrs.push_back(attr);
391 attr = nullptr;
392 }
393
394 var->unsupported_attr_dtype = temp_unsup_attr_dtype;
395 var->unsupported_attr_dspace = temp_unsup_attr_dspace;
396
397 if (!this->unsupported_var_attr_dspace && temp_unsup_attr_dspace)
398 this->unsupported_var_attr_dspace = true;
399 }
400
401 this->vars.push_back(var);
402 if (H5Dclose(cdset) < 0)
403 throw2("Error closing the HDF5 dataset ", full_path_name);
404 }
405 break;
406
407 case H5O_TYPE_NAMED_DATATYPE: {
408 // ignore the named datatype
409 if (true == include_attr && true == check_ignored) {
410 this->add_ignored_info_namedtypes(string(gname), string(oname.begin(), oname.end()));
411 }
412 }
413 break;
414 default:
415 break;
416 } // "switch (obj_type)"
417 } // try
418 catch (...) {
419
420 if (attr != nullptr) {
421 delete attr;
422 attr = nullptr;
423 }
424
425 if (var != nullptr) {
426 delete var;
427 var = nullptr;
428 }
429
430 if (group != nullptr) {
431 delete group;
432 group = nullptr;
433 }
434
435 if (cgroup != -1) H5Gclose(cgroup);
436
437 if (cdset != -1) H5Dclose(cdset);
438 throw;
439
440 } // catch
441 } // "for (hsize_t i = 0; i < nelems; i++)"
442
443}
444
445// Retrieve HDF5 dataset compression ratio
446float File::Retrieve_H5_VarCompRatio(const Var *var, const hid_t dset_id) const
447{
448
449 float comp_ratio = 1.0;
450 // Obtain the data type of the variable.
451 hid_t dset_create_plist = H5Dget_create_plist(dset_id);
452 if (dset_create_plist < 0)
453 throw1("unable to obtain hdf5 dataset creation property list ");
454 H5D_layout_t dset_layout = H5Pget_layout(dset_create_plist);
455 if (dset_layout < 0) {
456 H5Pclose(dset_create_plist);
457 throw1("unable to obtain hdf5 dataset creation property list storage layout");
458 }
459
460 if (dset_layout == H5D_CHUNKED) {
461
462 hsize_t dstorage_size = H5Dget_storage_size(dset_id);
463 if (dstorage_size > 0 && var->total_elems > 0) {
464 hid_t ty_id = -1;
465
466 // Obtain the data type of the variable.
467 if ((ty_id = H5Dget_type(dset_id)) < 0)
468 throw1("unable to obtain hdf5 datatype for the dataset ");
469 size_t type_size = H5Tget_size(ty_id);
470 comp_ratio = ((float)((var->total_elems) * type_size))/dstorage_size;
471 H5Tclose(ty_id);
472 }
473
474 }
475 H5Pclose(dset_create_plist);
476 return comp_ratio;
477
478}
479// Retrieve HDF5 dataset datatype
480void File::Retrieve_H5_VarType(Var *var, hid_t dset_id, const string & varname, bool &unsup_var_dtype)
481{
482
483 hid_t ty_id = -1;
484
485 // Obtain the data type of the variable.
486 if ((ty_id = H5Dget_type(dset_id)) < 0)
487 throw2("unable to obtain hdf5 datatype for the dataset ", varname);
488
489 // The following datatype class and datatype will not be supported for the CF option.
490 // H5T_TIME, H5T_BITFIELD
491 // H5T_OPAQUE, H5T_ENUM
492 // H5T_REFERENCE, H5T_COMPOUND
493 // H5T_VLEN,H5T_ARRAY
494 // 64-bit integer
495
496 // Note: H5T_REFERENCE H5T_COMPOUND and H5T_ARRAY can be mapped to DAP2 DDS for the default option.
497 // H5T_COMPOUND, H5T_ARRAY can be mapped to DAP2 DAS for the default option.
498 // 1-D variable length of string can also be mapped for the CF option..
499 // The variable length string class is H5T_STRING rather than H5T_VLEN,
500 // We also ignore the mapping of integer 64 bit since DAP2 doesn't
501 // support 64-bit integer. In theory, DAP2 doesn't support long double
502 // (128-bit or 92-bit floating point type).
503 //
504
505 var->dtype = HDF5CFUtil::H5type_to_H5DAPtype(ty_id);
506 if (false == HDF5CFUtil::cf_strict_support_type(var->dtype,_is_dap4))
507 unsup_var_dtype = true;
508
509 if (H5Tclose(ty_id) < 0)
510 throw1("Unable to close the HDF5 datatype ");;
511}
512
513// Retrieve the HDF5 dataset dimension information
514void File::Retrieve_H5_VarDim(Var *var, hid_t dset_id, const string & varname, bool &unsup_var_dspace)
515{
516
517 vector<hsize_t> dsize;
518 vector<hsize_t> maxsize;
519
520 hid_t dspace_id = -1;
521 hid_t ty_id = -1;
522
523 try {
524 if ((dspace_id = H5Dget_space(dset_id)) < 0)
525 throw2("Cannot get hdf5 dataspace id for the variable ", varname);
526
527 H5S_class_t space_class = H5S_NO_CLASS;
528 if ((space_class = H5Sget_simple_extent_type(dspace_id)) < 0)
529 throw2("Cannot obtain the HDF5 dataspace class for the variable ", varname);
530
531 if (H5S_NULL == space_class)
532 unsup_var_dspace = true;
533 else {
534 if (false == unsup_var_dspace) {
535
536 hssize_t h5_total_elms = H5Sget_simple_extent_npoints(dspace_id);
537 if (h5_total_elms < 0)
538 throw2("Cannot get the total number of elements of HDF5 dataset ", varname);
539 else
540 var->total_elems = (size_t) h5_total_elms;
541 int ndims = H5Sget_simple_extent_ndims(dspace_id);
542 if (ndims < 0)
543 throw2("Cannot get the hdf5 dataspace number of dimension for the variable ", varname);
544
545 var->rank = ndims;
546 if (ndims != 0) {
547 dsize.resize(ndims);
548 maxsize.resize(ndims);
549 }
550
551 // The netcdf DAP client supports the representation of the unlimited dimension.
552 // So we need to check.
553 if (H5Sget_simple_extent_dims(dspace_id, dsize.data(), maxsize.data()) < 0)
554 throw2("Cannot obtain the dim. info for the variable ", varname);
555
556 for (int i = 0; i < ndims; i++) {
557 auto dim = new Dimension(dsize[i]);
558 if (maxsize[i] == H5S_UNLIMITED) {
559 dim->unlimited_dim = true;
560 if (false == have_udim)
561 have_udim = true;
562 }
563 var->dims.push_back(dim);
564 }
565 }
566 }
567
568 var->unsupported_dspace = unsup_var_dspace;
569
570 if (H5Sclose(dspace_id) < 0)
571 throw1("Cannot close the HDF5 dataspace .");
572
573 }
574
575 catch (...) {
576
577 if (dspace_id != -1) H5Sclose(dspace_id);
578
579 if (ty_id != -1) H5Tclose(ty_id);
580 throw;
581 }
582
583}
584
585// Retrieve the HDF5 attribute information.
586void File::Retrieve_H5_Attr_Info(Attribute * attr, hid_t obj_id, const int j, bool &unsup_attr_dtype,
587 bool &unsup_attr_dspace)
588
589{
590
591 hid_t attrid = -1;
592 hid_t ty_id = -1;
593 hid_t aspace_id = -1;
594 hid_t memtype = -1;
595
596 try {
597
598 // Obtain the attribute ID.
599 if ((attrid = H5Aopen_by_idx(obj_id, ".", H5_INDEX_CRT_ORDER, H5_ITER_INC, (hsize_t) j, H5P_DEFAULT,
600 H5P_DEFAULT)) < 0)
601 throw1("Unable to open attribute by index ");
602
603 // Obtain the size of attribute name.
604 ssize_t name_size = H5Aget_name(attrid, 0, nullptr);
605 if (name_size < 0)
606 throw1("Unable to obtain the size of the hdf5 attribute name ");
607
608 string attr_name;
609 attr_name.resize(name_size + 1);
610
611 // Obtain the attribute name.
612 if ((H5Aget_name(attrid, name_size + 1, &attr_name[0])) < 0)
613 throw1("unable to obtain the hdf5 attribute name ");
614
615 // Obtain the type of the attribute.
616 if ((ty_id = H5Aget_type(attrid)) < 0)
617 throw2("unable to obtain hdf5 datatype for the attribute ", attr_name);
618
619 // The following datatype class and datatype will not be supported for the CF option.
620 // H5T_TIME, H5T_BITFIELD
621 // H5T_OPAQUE, H5T_ENUM
622 // H5T_REFERENCE, H5T_COMPOUND
623 // H5T_VLEN,H5T_ARRAY
624 // 64-bit integer
625
626 // Note: H5T_REFERENCE H5T_COMPOUND and H5T_ARRAY can be mapped to DAP2 DDS for the default option.
627 // H5T_COMPOUND, H5T_ARRAY can be mapped to DAP2 DAS for the default option.
628 // 1-D variable length of string can also be mapped for the CF option..
629 // The variable length string class is H5T_STRING rather than H5T_VLEN,
630 // We also ignore the mapping of integer 64 bit since DAP2 doesn't
631 // support 64-bit integer. In theory, DAP2 doesn't support long double
632 // (128-bit or 92-bit floating point type).
633 //
634 attr->dtype = HDF5CFUtil::H5type_to_H5DAPtype(ty_id);
635 if (false == HDF5CFUtil::cf_strict_support_type(attr->dtype,_is_dap4))
636 unsup_attr_dtype = true;
637
638 if(H5VSTRING == attr->dtype || H5FSTRING == attr->dtype) {
639 H5T_cset_t c_set_type = H5Tget_cset(ty_id);
640 if(c_set_type <0)
641 throw2("Cannot get hdf5 character set type for the attribute ", attr_name);
642 // This is a UTF-8 string
643 if(c_set_type == 1)
644 attr->is_cset_ascii = false;
645 }
646
647 if ((aspace_id = H5Aget_space(attrid)) < 0)
648 throw2("Cannot get hdf5 dataspace id for the attribute ", attr_name);
649
650 int ndims = H5Sget_simple_extent_ndims(aspace_id);
651 if (ndims < 0)
652 throw2("Cannot get the hdf5 dataspace number of dimension for attribute ", attr_name);
653
654 hsize_t nelmts = 1;
655
656 // if it is a scalar attribute, just define number of elements to be 1.
657 if (ndims != 0) {
658
659 vector<hsize_t> asize;
660 vector<hsize_t> maxsize;
661 asize.resize(ndims);
662 maxsize.resize(ndims);
663
664 // Obtain the attribute data space information.
665 if (H5Sget_simple_extent_dims(aspace_id, asize.data(), maxsize.data()) < 0)
666 throw2("Cannot obtain the dim. info for the attribute ", attr_name);
667
668 // Here we need to take care of 0-length attribute. This is legal in HDF5.
669 for (int dim_count = 0;dim_count < ndims; dim_count ++) {
670 // STOP adding unsupported_attr_dspace!
671 if (asize[dim_count] == 0) {
672 unsup_attr_dspace = true;
673 break;
674 }
675 }
676
677 if (false == unsup_attr_dspace) {
678 // Return ndims and size[ndims].
679 for (int dim_count = 0; dim_count< ndims; dim_count++)
680 nelmts *= asize[dim_count];
681 }
682 else
683 nelmts = 0;
684 } // "if(ndims != 0)"
685
686 size_t ty_size = H5Tget_size(ty_id);
687 if (0 == ty_size)
688 throw2("Cannot obtain the dtype size for the attribute ", attr_name);
689
690 memtype = H5Tget_native_type(ty_id, H5T_DIR_ASCEND);
691 if (memtype < 0)
692 throw2("Cannot obtain the memory datatype for the attribute ", attr_name);
693
694 // Store the name and the count
695 string temp_aname(attr_name.begin(), attr_name.end());
696 attr->name = temp_aname.substr(0, temp_aname.size() - 1);
697 attr->newname = attr->name;
698 attr->count = nelmts;
699
700 // Release HDF5 resources.
701 if (H5Tclose(ty_id) < 0)
702 throw1("Cannot successfully close the attribute datatype.");
703 if (H5Tclose(memtype) < 0)
704 throw1("Cannot successfully close the attribute memory datatype.");
705 if (H5Sclose(aspace_id) < 0)
706 throw1("Cannot successfully close the HDF5 dataspace.");
707 if (H5Aclose(attrid) < 0)
708 throw1("Cannot successfully close the HDF5 attribute.");
709
710 } // try
711 catch (...) {
712
713 if (ty_id != -1) H5Tclose(ty_id);
714
715 if (memtype != -1) H5Tclose(memtype);
716
717 if (aspace_id != -1) H5Sclose(aspace_id);
718
719 if (attrid != -1) H5Aclose(attrid);
720
721 throw;
722 }
723
724}
725
726// Retrieve all HDF5 supported attribute values.
728{
729
730 for (auto ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira)
731 Retrieve_H5_Attr_Value(*ira, "/");
732
733 for (auto irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
734 for (auto ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
735 Retrieve_H5_Attr_Value(*ira, (*irg)->path);
736 }
737 }
738
739 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
740 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
741 Retrieve_H5_Attr_Value(*ira, (*irv)->fullpath);
742 }
743 }
744}
745
747{
748 for (auto ira = var->attrs.begin(); ira != var->attrs.end(); ++ira) {
749 Retrieve_H5_Attr_Value(*ira, var->fullpath);
750 }
751}
752
753// Retrieve the values of a specific HDF5 attribute.
754void File::Retrieve_H5_Attr_Value(Attribute *attr, const string & obj_name)
755{
756
757 // Define HDF5 object Ids.
758 hid_t obj_id = -1;
759 hid_t attr_id = -1;
760 hid_t ty_id = -1;
761 hid_t memtype_id = -1;
762 hid_t aspace_id = -1;
763
764 try {
765
766 // Open the object that hold this attribute
767 obj_id = H5Oopen(this->fileid, obj_name.c_str(), H5P_DEFAULT);
768 if (obj_id < 0)
769 throw2("Cannot open the object ", obj_name);
770
771 attr_id = H5Aopen(obj_id, (attr->name).c_str(), H5P_DEFAULT);
772 if (attr_id < 0)
773 throw4("Cannot open the attribute ", attr->name, " of object ", obj_name);
774
775 ty_id = H5Aget_type(attr_id);
776 if (ty_id < 0)
777 throw4("Cannot obtain the datatype of the attribute ", attr->name, " of object ", obj_name);
778
779 memtype_id = H5Tget_native_type(ty_id, H5T_DIR_ASCEND);
780 if (memtype_id < 0)
781 throw2("Cannot obtain the memory datatype for the attribute ", attr->name);
782
783 size_t ty_size = H5Tget_size(memtype_id);
784 if (0 == ty_size)
785 throw4("Cannot obtain the dtype size for the attribute ", attr->name, " of object ", obj_name);
786
787 size_t total_bytes = attr->count * ty_size;
788
789 // We have to handle variable length string differently.
790 if (H5VSTRING == attr->dtype) {
791
792 // Variable length string attribute values only store pointers of the actual string value.
793 vector<char> temp_buf;
794 temp_buf.resize(total_bytes);
795
796 if (H5Aread(attr_id, memtype_id, temp_buf.data()) < 0)
797 throw4("Cannot obtain the value of the attribute ", attr->name, " of object ", obj_name);
798
799 char *temp_bp = nullptr;
800 char *ptr_1stvlen_ptr = temp_buf.data();
801 temp_bp = temp_buf.data();
802 char* onestring = nullptr;
803 string total_vstring = "";
804
805 attr->strsize.resize(attr->count);
806
807 for (unsigned int temp_i = 0; temp_i < attr->count; temp_i++) {
808
809 // This line will assure that we get the real variable length string value.
810 onestring = *(char **) temp_bp;
811 if (onestring != nullptr) {
812 total_vstring += string(onestring);
813 attr->strsize[temp_i] = (string(onestring)).size();
814 }
815 else
816 attr->strsize[temp_i] = 0;
817
818 // going to the next value.
819 temp_bp += ty_size;
820 }
821
822 if (ptr_1stvlen_ptr != nullptr) {
823 aspace_id = H5Aget_space(attr_id);
824 if (aspace_id < 0)
825 throw4("Cannot obtain space id for ", attr->name, " of object ", obj_name);
826
827 // Reclaim any VL memory if necessary.
828 if (H5Dvlen_reclaim(memtype_id, aspace_id, H5P_DEFAULT, temp_buf.data()) < 0)
829 throw4("Cannot reclaim VL memory for ", attr->name, " of object ", obj_name);
830
831 H5Sclose(aspace_id);
832 }
833
834 if (HDF5CFUtil::H5type_to_H5DAPtype(ty_id) != H5VSTRING)
835 throw4("Error to obtain the VL string type for attribute ", attr->name, " of object ", obj_name);
836
837 attr->value.resize(total_vstring.size());
838
839 copy(total_vstring.begin(), total_vstring.end(), attr->value.begin());
840
841 }
842 else {
843
844 if (attr->dtype == H5FSTRING) {
845 attr->fstrsize = ty_size;
846 }
847
848 attr->value.resize(total_bytes);
849
850 // Read HDF5 attribute data.
851 if (H5Aread(attr_id, memtype_id, (void *) &attr->value[0]) < 0)
852 throw4("Cannot obtain the dtype size for the attribute ", attr->name, " of object ", obj_name);
853
854 if (attr->dtype == H5FSTRING) {
855
856 size_t sect_size = ty_size;
857 int num_sect = 1;
858 if (sect_size > 0)
859 num_sect =
860 (total_bytes % sect_size == 0) ? (total_bytes / sect_size) : (total_bytes / sect_size + 1);
861 else
862 throw4("The attribute datatype size is not a positive integer ", attr->name, " of object ",
863 obj_name);
864
865 vector<size_t> sect_newsize;
866 sect_newsize.resize(num_sect);
867
868 auto total_fstring = string(attr->value.begin(), attr->value.end());
869
870 string new_total_fstring = HDF5CFUtil::trim_string(memtype_id, total_fstring, num_sect, sect_size,
871 sect_newsize);
872 attr->value.resize(new_total_fstring.size());
873 copy(new_total_fstring.begin(), new_total_fstring.end(), attr->value.begin());
874 attr->strsize.resize(num_sect);
875 for (int temp_i = 0; temp_i < num_sect; temp_i++)
876 attr->strsize[temp_i] = sect_newsize[temp_i];
877
878#if 0
879 // "h5","new string value " <<string(attr->value.begin(), attr->value.end()) <<endl;
880 cerr<<"Attr name is "<<attr->name <<endl;
881 for (int temp_i = 0; temp_i <num_sect; temp_i ++)
882 cerr<<"string new section size = " << attr->strsize[temp_i] <<endl;
883#endif
884 }
885 }
886
887 if (H5Tclose(memtype_id) < 0)
888 throw1("Fail to close the HDF5 memory datatype ID.");
889 if (H5Tclose(ty_id) < 0)
890 throw1("Fail to close the HDF5 datatype ID.");
891 if (H5Aclose(attr_id) < 0)
892 throw1("Fail to close the HDF5 attribute ID.");
893 if (H5Oclose(obj_id) < 0)
894 throw1("Fail to close the HDF5 object ID.");
895
896 }
897
898 catch (...) {
899
900 if (memtype_id != -1) H5Tclose(memtype_id);
901
902 if (ty_id != -1) H5Tclose(ty_id);
903
904 if (aspace_id != -1) H5Sclose(aspace_id);
905
906 if (attr_id != -1) H5Aclose(attr_id);
907
908 if (obj_id != -1) H5Oclose(obj_id);
909
910 throw;
911 }
912
913}
914
915// Handle the unsupported datatype
916void File::Handle_Unsupported_Dtype(bool include_attr)
917{
918
919 if (true == include_attr) {
920 Handle_Group_Unsupported_Dtype();
921 Handle_VarAttr_Unsupported_Dtype();
922 }
923
924 Handle_Var_Unsupported_Dtype();
925}
926
927//Leave this code here.
928#if 0
929// First the root attributes
930if (true == include_attr) {
931 if (true == this->unsupported_attr_dtype) {
932 for (vector<Attribute *>::iterator ira = this->root_attrs.begin();
933 ira != this->root_attrs.end(); ) {
934 H5DataType temp_dtype = (*ira)->getType();
935 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
936 delete (*ira);
937 ira = this->root_attrs.erase(ira);
938
939 }
940 else {
941 ++ira;
942 }
943 }
944 }
945}
946
947// Then the group attributes
948if (false == this->groups.empty()) {
949 for (vector<Group *>::iterator irg = this->groups.begin();
950 irg != this->groups.end(); ++irg) {
951 if (false == (*irg)->attrs.empty()) {
952 if (true == (*irg)->unsupported_attr_dtype) {
953 for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin();
954 ira != (*irg)->attrs.end(); ) {
955 H5DataType temp_dtype = (*ira)->getType();
956 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
957 delete (*ira);
958 ira = (*irg)->attrs.erase(ira);
959 }
960 else {
961 ++ira;
962 }
963 }
964 }
965 }
966 }
967}
968}
969
970 // Then the variable(HDF5 dataset) and the correponding attributes.
971if (false == this->vars.empty()) {
972if (true == include_attr) {
973 for (vector<Var *>::iterator irv = this->vars.begin();
974 irv != this->vars.end();++irv ) {
975 if (false == (*irv)->attrs.empty()) {
976 if (true == (*irv)->unsupported_attr_dtype) {
977 for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
978 ira != (*irv)->attrs.end(); ) {
979 H5DataType temp_dtype = (*ira)->getType();
980 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
981 delete (*ira);
982 ira = (*irv)->attrs.erase(ira);
983 //ira--;
984 }
985 else {
986 ++ira;
987 }
988 }
989 }
990 }
991 }
992}
993if (true == this->unsupported_var_dtype) {
994 // "h5","having unsupported variable datatype" <<endl;
995 for (vector<Var *>::iterator irv = this->vars.begin();
996 irv != this->vars.end(); ) {
997 H5DataType temp_dtype = (*irv)->getType();
998 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
999 delete (*irv);
1000 irv = this->vars.erase(irv);
1001 //irv--;
1002 }
1003 else
1004 ++irv;
1005 }
1006}
1007}
1008#endif
1009
1010void File::Handle_Group_Unsupported_Dtype()
1011{
1012
1013 // First root
1014 if (false == this->root_attrs.empty()) {
1015 if (true == this->unsupported_attr_dtype) {
1016 for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end();) {
1017 H5DataType temp_dtype = (*ira)->getType();
1018 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
1019 delete (*ira);
1020 ira = this->root_attrs.erase(ira);
1021 }
1022 else {
1023 ++ira;
1024 }
1025 }
1026 }
1027 }
1028
1029 // Then the group attributes
1030 if (false == this->groups.empty()) {
1031 for (auto irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1032 if (false == (*irg)->attrs.empty()) {
1033 if (true == (*irg)->unsupported_attr_dtype) {
1034 for (auto ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end();) {
1035 H5DataType temp_dtype = (*ira)->getType();
1036 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
1037 delete (*ira);
1038 ira = (*irg)->attrs.erase(ira);
1039 }
1040 else {
1041 ++ira;
1042 }
1043 }
1044 }
1045 }
1046 }
1047 }
1048}
1049
1050// Generate group unsupported datatype Information, this is for the BES ignored object key
1051void File::Gen_Group_Unsupported_Dtype_Info()
1052{
1053
1054 // First root
1055 if (false == this->root_attrs.empty()) {
1056 //if (true == this->unsupported_attr_dtype) {
1057 for (auto ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
1058 H5DataType temp_dtype = (*ira)->getType();
1059 // TODO: Don't know why we still include 64-bit integer here.
1060 //if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype) || temp_dtype == H5INT64 || temp_dtype == H5UINT64) {
1061 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)
1062 || temp_dtype == H5INT64 || temp_dtype == H5UINT64) {
1063 this->add_ignored_info_attrs(true, "/", (*ira)->name);
1064 }
1065 }
1066 //}
1067 }
1068
1069 // Then the group attributes
1070 if (false == this->groups.empty()) {
1071 for (auto irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1072 if (false == (*irg)->attrs.empty()) {
1073 //if (true == (*irg)->unsupported_attr_dtype) {
1074 for (auto ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
1075 H5DataType temp_dtype = (*ira)->getType();
1076 // TODO: Don't know why we still include 64-bit integer here.
1077 //if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype) || temp_dtype == H5INT64 || temp_dtype==H5UINT64 ) {
1078 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)
1079 || temp_dtype == H5INT64 || temp_dtype==H5UINT64 ) {
1080 this->add_ignored_info_attrs(true, (*irg)->path, (*ira)->name);
1081 }
1082 }
1083 //}
1084 }
1085 }
1086 }
1087}
1088
1089// Handler unsupported variable datatype
1090void File::Handle_Var_Unsupported_Dtype()
1091{
1092 if (false == this->vars.empty()) {
1093 if (true == this->unsupported_var_dtype) {
1094 for (auto irv = this->vars.begin(); irv != this->vars.end();) {
1095 H5DataType temp_dtype = (*irv)->getType();
1096 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
1097 delete (*irv);
1098 irv = this->vars.erase(irv);
1099 }
1100 else {
1101 ++irv;
1102
1103 }
1104 }
1105 }
1106 }
1107}
1108
1109// Generate unsupported variable type info. This is for the ignored objects.
1110void File::Gen_Var_Unsupported_Dtype_Info()
1111{
1112
1113 if (false == this->vars.empty()) {
1114 //if (true == this->unsupported_var_dtype) {
1115 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1116 H5DataType temp_dtype = (*irv)->getType();
1117 //TODO: don't know why 64-bit integer is still listed here.
1118 //if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)||(H5INT64 == temp_dtype) ||(H5UINT64 == temp_dtype)) {
1119 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)
1120 ||(H5INT64 == temp_dtype) ||(H5UINT64 == temp_dtype)) {
1121 this->add_ignored_info_objs(false, (*irv)->fullpath);
1122 }
1123 }
1124 //}
1125 }
1126
1127}
1128
1129// Handling unsupported datatypes for variable(HDF5 dataset) and the correponding attributes.
1130void File::Handle_VarAttr_Unsupported_Dtype()
1131{
1132 if (false == this->vars.empty()) {
1133 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1134 if (false == (*irv)->attrs.empty()) {
1135 if (true == (*irv)->unsupported_attr_dtype) {
1136 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();) {
1137 H5DataType temp_dtype = (*ira)->getType();
1138 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
1139 delete (*ira);
1140 ira = (*irv)->attrs.erase(ira);
1141 }
1142 else {
1143 ++ira;
1144 }
1145 }
1146 }
1147 }
1148 }
1149 }
1150}
1151
1152// Generated unsupported var/attribute unsupported datatype Info when the BES ignored object key is on.
1153void File::Gen_VarAttr_Unsupported_Dtype_Info()
1154{
1155
1156 if (false == this->vars.empty()) {
1157 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1158 if (false == (*irv)->attrs.empty()) {
1159 //if (true == (*irv)->unsupported_attr_dtype) {
1160 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1161 H5DataType temp_dtype = (*ira)->getType();
1162 // TODO: check why 64-bit integer is still listed here.
1163 //if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype) || (temp_dtype==H5INT64) || (temp_dtype == H5UINT64)) {
1164 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)
1165 || (temp_dtype==H5INT64) || (temp_dtype == H5UINT64)) {
1166 this->add_ignored_info_attrs(false, (*irv)->fullpath, (*ira)->name);
1167 }
1168 }
1169 //}
1170 }
1171 }
1172 }
1173}
1174
1175// Generated unsupported datatype information for HDF5 dimension scales.
1176// The datatypes of HDF5 dimension scales("DIMENSION_LIST" and "REFERENCE_LIST")
1177// are not supported. However, the information
1178// are retrieved by the handlers so we don't want to report them as ignored objects.
1179void File::Gen_DimScale_VarAttr_Unsupported_Dtype_Info()
1180{
1181
1182 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1183
1184 // If the attribute REFERENCE_LIST comes with the attribut CLASS, the
1185 // attribute REFERENCE_LIST is okay to ignore. No need to report.
1186 bool is_ignored = ignored_dimscale_ref_list((*irv));
1187 if (false == (*irv)->attrs.empty()) {
1188 //if (true == (*irv)->unsupported_attr_dtype) {
1189 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1190 H5DataType temp_dtype = (*ira)->getType();
1191 // TODO: check why 64-bit is still listed here.
1192 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)
1193 || (temp_dtype == H5INT64) || (temp_dtype == H5UINT64)) {
1194 // "DIMENSION_LIST" is okay to ignore and "REFERENCE_LIST"
1195 // is okay to ignore if the variable has another attribute
1196 // CLASS="DIMENSION_SCALE"
1197 if (("DIMENSION_LIST" != (*ira)->name)
1198 && ("REFERENCE_LIST" != (*ira)->name || true == is_ignored))
1199 this->add_ignored_info_attrs(false, (*irv)->fullpath, (*ira)->name);
1200 }
1201 }
1202 //}
1203 }
1204 }
1205}
1206
1207// Handle unsupported dataspace for group attributes.
1208void File::Handle_GroupAttr_Unsupported_Dspace()
1209{
1210
1211 // First root
1212 if (false == this->root_attrs.empty()) {
1213 if (true == this->unsupported_attr_dspace) {
1214 for (auto ira = this->root_attrs.begin(); ira != this->root_attrs.end();) {
1215 // Remove 0-size attribute
1216 if ((*ira)->count == 0) {
1217 delete (*ira);
1218 ira = this->root_attrs.erase(ira);
1219 }
1220 else {
1221 ++ira;
1222 }
1223 }
1224 }
1225 }
1226
1227 // Then the group attributes
1228 if (false == this->groups.empty()) {
1229 for (auto irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1230 if (false == (*irg)->attrs.empty()) {
1231 if (true == (*irg)->unsupported_attr_dspace) {
1232 for (auto ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end();) {
1233 if ((*ira)->count == 0) {
1234 delete (*ira);
1235 ira = (*irg)->attrs.erase(ira);
1236 }
1237 else {
1238 ++ira;
1239 }
1240 }
1241 }
1242 }
1243 }
1244 }
1245}
1246
1247// Handle unsupported data space information for variable and attribute
1248void File::Handle_VarAttr_Unsupported_Dspace()
1249{
1250
1251 if (false == this->vars.empty()) {
1252 if (true == this->unsupported_var_attr_dspace) {
1253 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1254 if (false == (*irv)->attrs.empty()) {
1255 if (true == (*irv)->unsupported_attr_dspace) {
1256 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();) {
1257 if (0 == (*ira)->count) {
1258 delete (*ira);
1259 ira = (*irv)->attrs.erase(ira);
1260 }
1261 else {
1262 ++ira;
1263 }
1264 }
1265 }
1266 }
1267 }
1268 }
1269 }
1270}
1271
1272// Handle unsupported data space.
1273void File::Handle_Unsupported_Dspace(bool include_attr)
1274{
1275
1276 // The unsupported data space
1277 if (false == this->vars.empty()) {
1278 if (true == this->unsupported_var_dspace) {
1279 for (auto irv = this->vars.begin(); irv != this->vars.end();) {
1280 if (true == (*irv)->unsupported_dspace) {
1281 delete (*irv);
1282 irv = this->vars.erase(irv);
1283 }
1284 else {
1285 ++irv;
1286
1287 }
1288 }
1289 }
1290 }
1291
1292 if (true == include_attr) {
1293 Handle_GroupAttr_Unsupported_Dspace();
1294 Handle_VarAttr_Unsupported_Dspace();
1295 }
1296}
1297
1298// Generated unsupported dataspace Info when the BES ignored object key is on.
1299void File::Gen_Unsupported_Dspace_Info()
1300{
1301
1302 // Notice in this function, we deliberately don't put the case when an attribute dimension has 0 length.
1303 // Since doing this requires non-trivial change of the source code and the 0-size attribute case is really, really rare,
1304 // so we just "ignore" this case in the "ignored" information.
1305 // In fact, the zero size variable is allowed in both HDF5 and DAP2. So we don't ignore 0-size HDF5 dataset. So
1306 // the only case this function checks is the H5S_NULL case.
1307 if (false == this->vars.empty()) {
1308 if (true == this->unsupported_var_dspace) {
1309 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1310 if (true == (*irv)->unsupported_dspace) {
1311 this->add_ignored_info_objs(true, (*irv)->fullpath);
1312 }
1313 }
1314 }
1315 }
1316
1317}
1318
1319// Handle other unsupported information.
1320void File::Handle_Unsupported_Others(bool include_attr)
1321{
1322
1323 if (true == this->check_ignored && true == include_attr) {
1324
1325 if (true == HDF5RequestHandler::get_drop_long_string()) {
1326
1327 // netCDF java doesn't have limitation for attributes
1328#if 0
1329 for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
1330 if (H5FSTRING == (*ira)->dtype || H5VSTRING == (*ira)->dtype) {
1331 if ((*ira)->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
1332 this->add_ignored_droplongstr_hdr();
1333 this->add_ignored_grp_longstr_info("/", (*ira)->name);
1334 }
1335 }
1336 }
1337
1338 for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1339 for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
1340 if (H5FSTRING == (*ira)->dtype || H5VSTRING == (*ira)->dtype) {
1341 if ((*ira)->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
1342 this->add_ignored_droplongstr_hdr();
1343 this->add_ignored_grp_longstr_info((*irg)->path, (*ira)->name);
1344 }
1345 }
1346 }
1347 }
1348#endif
1349 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1350 if (true == Check_DropLongStr((*irv), nullptr)) {
1351 this->add_ignored_droplongstr_hdr();
1352 this->add_ignored_var_longstr_info((*irv), nullptr);
1353 }
1354 // netCDF java doesn't have limitation for attributes
1355#if 0
1356 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1357 if (true == Check_DropLongStr((*irv), (*ira))) {
1358 this->add_ignored_droplongstr_hdr();
1359 this->add_ignored_var_longstr_info((*irv), (*ira));
1360 }
1361 }
1362#endif
1363 }
1364 }
1365 }
1366
1367}
1368
1369// Flatten the object name, mainly call get_CF_string.
1370void File::Flatten_Obj_Name(bool include_attr)
1371{
1372
1373 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1374 (*irv)->newname = get_CF_string((*irv)->newname);
1375
1376 for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
1377 (*ird)->newname = get_CF_string((*ird)->newname);
1378 }
1379 }
1380
1381 if (true == include_attr) {
1382
1383 for (auto ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
1384 (*ira)->newname = get_CF_string((*ira)->newname);
1385 }
1386
1387 for (auto irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1388 (*irg)->newname = get_CF_string((*irg)->newname);
1389 for (auto ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
1390 (*ira)->newname = get_CF_string((*ira)->newname);
1391 }
1392 }
1393
1394 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1395 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1396 (*ira)->newname = get_CF_string((*ira)->newname);
1397 }
1398 }
1399 } // "if (true == include_attr)"
1400}
1401
1402// Variable name clashing
1403void File::Handle_Var_NameClashing(set<string>&objnameset)
1404{
1405
1406 Handle_General_NameClashing(objnameset, this->vars);
1407}
1408
1409// Group name clashing
1410void File::Handle_Group_NameClashing(set<string> &objnameset)
1411{
1412
1413 pair<set<string>::iterator, bool> setret;
1414
1415 // Now for DAS, we need to handle name clashings for
1416 // DAS tables. Namely we need to make sure the global attribute
1417 // table(HDF5_GLOBAL) and the attribute tables mapped from
1418 // HDF5 groups will not have name clashings with the variable name
1419 // lists. If having the name clashings, the global attribute table and the
1420 // the attribute tables generated from the groups will be changed.
1421 // The file attribute name clashing
1422
1423 setret = objnameset.insert(FILE_ATTR_TABLE_NAME);
1424 if (false == setret.second) {
1425
1426 int clash_index = 1;
1427 string fa_clash_name = FILE_ATTR_TABLE_NAME;
1428 HDF5CFUtil::gen_unique_name(fa_clash_name, objnameset, clash_index);
1429 FILE_ATTR_TABLE_NAME = fa_clash_name;
1430 }
1431
1432 // The group attribute name clashing
1433 Handle_General_NameClashing(objnameset, this->groups);
1434
1435}
1436
1437//Object attribute name clashing
1438void File::Handle_Obj_AttrNameClashing()
1439{
1440
1441 // Now handling the possible name clashings for attributes
1442 // For attribute clashings, we only need to resolve the name clashings
1443 // for attributes within each variable, file attributes and attributes
1444 // within each group. The name clashings for attributes should be very rare.
1445 // Potentially the checking and the correcting may be costly.
1446 // This is another reason for special products, we may not even need to check
1447 // the name clashings. KY 2011-12-24
1448
1449 set<string> objnameset;
1450
1451 // For root attributes
1452 Handle_General_NameClashing(objnameset, this->root_attrs);
1453
1454 // For group attributes
1455 for (auto irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1456 objnameset.clear();
1457 Handle_General_NameClashing(objnameset, (*irg)->attrs);
1458 }
1459
1460 // For variable attributes
1461 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1462 objnameset.clear();
1463 Handle_General_NameClashing(objnameset, (*irv)->attrs);
1464 }
1465}
1466
1467// Handle General name clashing
1468//class T must have member string newname. In our case, T is either groups, attributes or vars.
1469template<class T> void File::Handle_General_NameClashing(set<string>&objnameset, vector<T*>& objvec)
1470{
1471
1472#if 0
1473// set<string> objnameset;
1474#endif
1475
1476 pair<set<string>::iterator, bool> setret;
1477 set<string>::iterator iss;
1478
1479 vector<string> clashnamelist;
1480 vector<string>::iterator ivs;
1481
1482 map<int, int> cl_to_ol;
1483 int ol_index = 0;
1484 int cl_index = 0;
1485
1486 /*class*/
1487 typename vector<T*>::iterator irv;
1488
1489 for (irv = objvec.begin(); irv != objvec.end(); ++irv) {
1490 setret = objnameset.insert((*irv)->newname);
1491 if (false == setret.second) {
1492 clashnamelist.insert(clashnamelist.end(), (*irv)->newname);
1493 cl_to_ol[cl_index] = ol_index;
1494 cl_index++;
1495 }
1496 ol_index++;
1497 }
1498
1499 // Now change the clashed elements to unique elements,
1500 // Generate the set which has the same size as the original vector.
1501 for (ivs = clashnamelist.begin(); ivs != clashnamelist.end(); ivs++) {
1502 int clash_index = 1;
1503 string temp_clashname = *ivs + '_';
1504 HDF5CFUtil::gen_unique_name(temp_clashname, objnameset, clash_index);
1505 *ivs = temp_clashname;
1506 }
1507
1508 // Now go back to the original vector, make it unique.
1509 for (unsigned int i = 0; i < clashnamelist.size(); i++)
1510 objvec[cl_to_ol[i]]->newname = clashnamelist[i];
1511
1512}
1513
1514// Handle General object name clashing
1515void File::Handle_GeneralObj_NameClashing(bool include_attr, set<string>& objnameset)
1516{
1517
1518 Handle_Var_NameClashing(objnameset);
1519 if (true == include_attr) {
1520 Handle_Group_NameClashing(objnameset);
1521 Handle_Obj_AttrNameClashing();
1522 }
1523}
1524
1525// Get CF name, flatten the path, change the non-alphanumeric letters to underscore.
1526string File::get_CF_string(string s)
1527{
1528
1529 if ("" == s) return s;
1530 string insertString(1, '_');
1531
1532 // Always start with _ if the first character is not a letter
1533 if (true == isdigit(s[0])) s.insert(0, insertString);
1534
1535 for (unsigned int i = 0; i < s.length(); i++)
1536 if ((false == isalnum(s[i])) && (s[i] != '_')) s[i] = '_';
1537
1538 return s;
1539
1540}
1541
1542// For the connection of variable dimensions. Build dimname to dimsize(unlimited) maps
1543void File::Insert_One_NameSizeMap_Element(string name, hsize_t size, bool unlimited)
1544{
1545 pair<map<string, hsize_t>::iterator, bool> mapret;
1546 mapret = dimname_to_dimsize.insert(pair<string, hsize_t>(name, size));
1547 if (false == mapret.second)
1548 throw4("The dimension name ", name, " should map to ", size);
1549
1550 pair<map<string, bool>::iterator, bool> mapret2;
1551 mapret2 = dimname_to_unlimited.insert(pair<string, bool>(name, unlimited));
1552 if (false == mapret2.second)
1553 throw3("The dimension name ", name, " unlimited dimension info. should be provided.");
1554
1555}
1556
1557// Similar to Inset_One_NameSizeMap_Element but the maps are provided as parameters.
1558void File::Insert_One_NameSizeMap_Element2(map<string, hsize_t>& name_to_size, map<string, bool>& name_to_unlimited,
1559 string name, hsize_t size, bool unlimited)
1560{
1561 pair<map<string, hsize_t>::iterator, bool> mapret;
1562 mapret = name_to_size.insert(pair<string, hsize_t>(name, size));
1563 if (false == mapret.second)
1564 throw4("The dimension name ", name, " should map to ", size);
1565
1566 pair<map<string, bool>::iterator, bool> mapret2;
1567 mapret2 = name_to_unlimited.insert(pair<string, bool>(name, unlimited));
1568 if (false == mapret2.second)
1569 throw3("The dimension name ", name, " unlimited dimension info. should be provided.");
1570
1571}
1572
1573// For dimension names added by the handlers, by default,
1574// Each dimension will have a unique dimension name. For example,
1575// Int foo[100][200] will be Int foo[Fakedim1][Fakedim2]
1576// If you have many variables, the dimension names may be too many.
1577// To reduce numbers, we ASSUME that the dimension having the same
1578// size shares the same dimension. In this way, the number of dimension names
1579// will be reduced.
1580// For example, Int foo2[100][300] will be Int foo2[Fakedim1][Fakedim3]
1581// instead of foo2[Fakedim3][Fakedim4]. However, that may impose
1582// another problem. Suppose Int Foosame[100][100] becomes
1583// Int Foosame[FakeDim1][FakeDim1]. This doesn't make sense for some
1584// applications. The fuction Adjust_Duplicate_FakeDim_Name will make sure
1585// this case will not happen.
1586void File::Add_One_FakeDim_Name(Dimension *dim)
1587{
1588
1589 stringstream sfakedimindex;
1590 string fakedimstr = "FakeDim";
1591 pair<set<string>::iterator, bool> setret;
1592 map<hsize_t, string>::iterator im;
1593 pair<map<hsize_t, string>::iterator, bool> mapret;
1594
1595 sfakedimindex << addeddimindex;
1596 string added_dimname = fakedimstr + sfakedimindex.str();
1597
1598 // Build up the size to fakedim map.
1599 mapret = dimsize_to_fakedimname.insert(pair<hsize_t, string>(dim->size, added_dimname));
1600 if (false == mapret.second) { //The dim size exists, use the corresponding name.
1601 dim->name = dimsize_to_fakedimname[dim->size];
1602 dim->newname = dim->name;
1603 }
1604 else { // Insert this (dimsize,dimname) pair to dimsize_to_fakedimname map successfully.
1605 //First make sure this new dim name doesn't have name clashing
1606 // with previous dim names, after the checking, inserting to the
1607 // dimname list set.
1608 // dimnamelist is a private memeber of File.
1609 setret = dimnamelist.insert(added_dimname);
1610 if (false == setret.second) {
1611 int clash_index = 1;
1612 string temp_clashname = added_dimname + '_';
1613 HDF5CFUtil::gen_unique_name(temp_clashname, dimnamelist, clash_index);
1614 dim->name = temp_clashname;
1615 dim->newname = dim->name;
1616 setret = dimnamelist.insert(dim->name);
1617 if (false == setret.second)
1618 throw2("Fail to insert the unique dimsizede name ", dim->name);
1619
1620 // We have to adjust the dim. name of the dimsize_to_fakedimname map, since the
1621 // dimname has been updated for this size.
1622 dimsize_to_fakedimname.erase(dim->size);
1623 mapret = dimsize_to_fakedimname.insert(pair<hsize_t, string>(dim->size, dim->name));
1624 if (false == mapret.second)
1625 throw4("The dimension size ", dim->size, " should map to ", dim->name);
1626 } // "if(false == setret.second)"
1627
1628 // New dim name is inserted successfully, update the dimname_to_dimsize map.
1629 dim->name = added_dimname;
1630 dim->newname = dim->name;
1631 Insert_One_NameSizeMap_Element(dim->name, dim->size, dim->unlimited_dim);
1632
1633 // Increase the dimindex since the new dimname has been inserted.
1634 addeddimindex++;
1635 } // else
1636}
1637
1638// See the function comments of Add_One_FakeDim_Name
1639void File::Adjust_Duplicate_FakeDim_Name(Dimension * dim)
1640{
1641
1642 // No need to adjust the dimsize_to_fakedimname map, only create a new Fakedim
1643 // The simplest way is to increase the dim index and resolve any name clashings with other dim names.
1644 // Note: No need to update the dimsize_to_dimname map since the original "FakeDim??" of this size
1645 // can be used as a dimension name of other variables. But we need to update the dimname_to_dimsize map
1646 // since this is a new dim name.
1647 stringstream sfakedimindex;
1648 pair<set<string>::iterator, bool> setret;
1649
1650 addeddimindex++;
1651 sfakedimindex << addeddimindex;
1652 string added_dimname = "FakeDim" + sfakedimindex.str();
1653 setret = dimnamelist.insert(added_dimname);
1654 if (false == setret.second) {
1655 int clash_index = 1;
1656 string temp_clashname = added_dimname + '_';
1657 HDF5CFUtil::gen_unique_name(temp_clashname, dimnamelist, clash_index);
1658 dim->name = temp_clashname;
1659 dim->newname = dim->name;
1660 setret = dimnamelist.insert(dim->name);
1661 if (false == setret.second)
1662 throw2("Fail to insert the unique dimsizede name ", dim->name);
1663 }
1664 dim->name = added_dimname;
1665 dim->newname = dim->name;
1666 Insert_One_NameSizeMap_Element(dim->name, dim->size, dim->unlimited_dim);
1667
1668 // Need to prepare for the next unique FakeDim.
1669 addeddimindex++;
1670}
1671// 1. See the function comments of Add_One_FakeDim_Name
1672// 2. This is for the case when we have foo[100][100],foo2[100][100].
1673// The Adjust_Duplicate_FakeDim_Name() assures the uniqueness of Dimension
1674// names for each var,like foo[FakeDim0][FakeDim1],foo2[FakeDim0][FakeDim3]..
1675// but the dimension names are not desirable for products like SMAP level 3.
1676// For similar vars, the dimension names like foo[FakeDim0][FakeDim1] and
1677// foo2[FakeDim0][FakeDim1] ,foo3[FakeDim0][FakeDim1][FakeDim2] are the way to go.
1678
1679void File::Adjust_Duplicate_FakeDim_Name2(Dimension * dim, int dup_dim_size_index)
1680{
1681
1682 // Search if vector of dup (dimsize dimname) pair has the current size
1683 // If yes, insert the dim name from the vector.
1684 // Note: in case we have foo[100][100][100] etc., we need to remember the
1685 // vector index, to obtain the correct dup_dim_size, dup_dim_name pair.
1686 // if no, build up the new FakeDim.
1687 bool dup_dim_size_exist = false;
1688 int temp_dup_dim_size_index = 0;
1689 for (unsigned i = 0; i <dup_dimsize_dimname.size();i++) {
1690 // The dup vector may include different size, so we need to check
1691 // if having the same size.
1692 if(dim->size == dup_dimsize_dimname[i].first) {
1693 temp_dup_dim_size_index++;
1694 // Make sure we obtain the correct index in the vector
1695 if(dup_dim_size_index == temp_dup_dim_size_index) {
1696 dup_dim_size_exist = true;
1697 dim->name = dup_dimsize_dimname[i].second;
1698 dim->newname = dim->name;
1699 break;
1700 }
1701 }
1702 }
1703
1704 // If we cannot find this dimension in the existing (dup dimsize dimname) vector,
1705 // create the FakeDim by increasing the index and update the dup vector etc.
1706 if(dup_dim_size_exist == false) {
1707
1708 stringstream sfakedimindex;
1709 pair<set<string>::iterator, bool> setret;
1710
1711 sfakedimindex << addeddimindex;
1712 string added_dimname = "FakeDim" + sfakedimindex.str();
1713 setret = dimnamelist.insert(added_dimname);
1714 if (false == setret.second) {
1715 throw2("Inside Adjust_Duplicate_FakeDim_Name2(), Fail to insert the unique dim name ", dim->name);
1716 }
1717 dim->name = added_dimname;
1718 dim->newname = dim->name;
1719 Insert_One_NameSizeMap_Element(dim->name, dim->size, dim->unlimited_dim);
1720 // push back to the duplicate size, name vector.
1721 dup_dimsize_dimname.push_back(make_pair(dim->size,dim->name));
1722 addeddimindex++;
1723
1724 }
1725
1726}
1727
1728// Replace all dimension names, this function is currently not used. So comment out. May delete it in the future.
1729#if 0
1730void File::Replace_Dim_Name_All(const string orig_dim_name, const string new_dim_name) {
1731
1732 // The newname of the original dimension should also be replaced by new_dim_name
1733 for (vector<Var *>::iterator irv = this->vars.begin();
1734 irv != this->vars.end(); ++irv) {
1735 for (vector<Dimension *>::iterator ird= (*irv)->dims.begin();
1736 ird != (*irv)->dims.end(); ++ird) {
1737 if((*ird)->name == orig_dim_name) {
1738 (*ird)->name = new_dim_name;
1739 (*ird)->newname = new_dim_name;
1740 }
1741
1742 }
1743 }
1744}
1745#endif
1746
1747#if 0
1748void File::Use_Dim_Name_With_Size_All(const string dim_name, const size_t dim_size) {
1749
1750 // The newname of the original dimension should also be replaced by new_dim_name
1751 for (vector<Var *>::iterator irv = this->vars.begin();
1752 irv != this->vars.end(); ++irv) {
1753 for (vector<Dimension *>::iterator ird= (*irv)->dims.begin();
1754 ird != (*irv)->dims.end(); ++ird) {
1755 if((*ird)->size == orig_dim_name) {
1756 (*ird)->name = new_dim_name;
1757 (*ird)->newname = new_dim_name;
1758 }
1759
1760 }
1761 }
1762}
1763#endif
1764
1765// Often times we need to add a CF attribute with string datatype because some products don't provide them
1766// Examples are units, comment etc.
1767void File::Add_Str_Attr(Attribute* attr, const string &attrname, const string& strvalue)
1768{
1769
1770 attr->name = attrname;
1771 attr->newname = attr->name;
1772 attr->dtype = H5FSTRING;
1773 attr->count = 1;
1774 attr->fstrsize = strvalue.size();
1775 attr->strsize.resize(1);
1776 attr->strsize[0] = attr->fstrsize;
1777 attr->value.resize(strvalue.size());
1778 copy(strvalue.begin(), strvalue.end(), attr->value.begin());
1779}
1780
1781#if 0
1782bool
1783File:: Var_Has_Attr(Var*var,const string &attrname) {
1784
1785 for (vector<Attribute *>:: iterator ira =var->attrs.begin(); ira !=var->attrs.end(); ++ira) {
1786
1787 // We only check the original attribute name
1788 // Remove the original "coordinates" attribute.
1789 if((*ira)->name == attrname || (*ira)->newname == attrname) {
1790 return true;
1791 }
1792 }
1793 return false;
1794}
1795#endif
1796
1797// Rretrieve the variable attribute in string.var_path is the variable path.
1798string File::Retrieve_Str_Attr_Value(Attribute *attr, const string & var_path)
1799{
1800
1801 if (attr != nullptr && var_path != "") {
1802 Retrieve_H5_Attr_Value(attr, var_path);
1803 string orig_attr_value(attr->value.begin(), attr->value.end());
1804 return orig_attr_value;
1805 }
1806 return "";
1807
1808}
1809
1810//Check if the attribute value of this variable is the input value.
1811bool File::Is_Str_Attr(Attribute* attr, const string& varfullpath, const string &attrname, const string& strvalue)
1812{
1813 bool ret_value = false;
1814 if (attrname == get_CF_string(attr->newname)) {
1815 Retrieve_H5_Attr_Value(attr, varfullpath);
1816 string attr_value(attr->value.begin(), attr->value.end());
1817 if (attr_value == strvalue) ret_value = true;
1818 }
1819 return ret_value;
1820}
1821
1822// If this latitude or longitude units follows the CF
1823bool File::has_latlon_cf_units(Attribute *attr, const string &varfullpath, bool is_lat)
1824{
1825 string attr_name = "units";
1826 if (true == is_lat) {
1827 string lat_unit_value = "degrees_north";
1828 return Is_Str_Attr(attr, varfullpath, attr_name, lat_unit_value);
1829 }
1830 else {
1831 string lon_unit_value = "degrees_east";
1832 return Is_Str_Attr(attr, varfullpath, attr_name, lon_unit_value);
1833 }
1834}
1835
1836// This function is mainly to add _FillValue.
1837void File::Add_One_Float_Attr(Attribute* attr, const string &attrname, float float_value)
1838{
1839 attr->name = attrname;
1840 attr->newname = attr->name;
1841 attr->dtype = H5FLOAT32;
1842 attr->count = 1;
1843 attr->value.resize(sizeof(float));
1844 memcpy(&(attr->value[0]), (void*) (&float_value), sizeof(float));
1845}
1846
1847// Products like GPM use string type for MissingValue, we need to change them to the corresponding variable datatype and
1848// get the value corrected.
1849void File::Change_Attr_One_Str_to_Others(Attribute* attr, const Var*var)
1850{
1851
1852 char *pEnd;
1853 // string to long int number.
1854 long int num_sli = 0;
1855 if (attr->dtype != H5FSTRING)
1856 throw2("Currently we only convert fixed-size string to other datatypes. ", attr->name);
1857 if (attr->count != 1)
1858 throw4("The fixed-size string count must be 1 and the current count is ", attr->count, " for the attribute ",
1859 attr->name);
1860
1861 Retrieve_H5_Attr_Value(attr, var->fullpath);
1862 string attr_value;
1863 attr_value.resize(attr->value.size());
1864 copy(attr->value.begin(), attr->value.end(), attr_value.begin());
1865
1866 switch (var->dtype) {
1867
1868 case H5UCHAR: {
1869 num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1870 if (num_sli < 0 || num_sli > UCHAR_MAX)
1871 throw5("Attribute type is unsigned char, the current attribute ", attr->name, " has the value ", num_sli,
1872 ". It is overflowed. ");
1873 else {
1874 unsigned char num_suc = (unsigned char) num_sli;
1875 attr->dtype = H5UCHAR;
1876 attr->value.resize(sizeof(unsigned char));
1877 memcpy(&(attr->value[0]), (void*) (&num_suc), sizeof(unsigned char));
1878 }
1879
1880 }
1881 break;
1882 case H5CHAR: {
1883 num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1884 if (num_sli < SCHAR_MIN || num_sli > SCHAR_MAX)
1885 throw5("Attribute type is signed char, the current attribute ", attr->name, " has the value ", num_sli,
1886 ". It is overflowed. ");
1887 else {
1888 char num_sc = (char) num_sli;
1889 attr->dtype = H5CHAR;
1890 attr->value.resize(sizeof(char));
1891 memcpy(&(attr->value[0]), (void*) (&num_sc), sizeof(char));
1892 }
1893
1894 }
1895 break;
1896 case H5INT16: {
1897 num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1898 if (num_sli < SHRT_MIN || num_sli > SHRT_MAX)
1899 throw5("Attribute type is 16-bit integer, the current attribute ", attr->name, " has the value ", num_sli,
1900 ". It is overflowed. ");
1901 else {
1902 short num_ss = (short) num_sli;
1903 attr->dtype = H5INT16;
1904 attr->value.resize(sizeof(short));
1905 memcpy(&(attr->value[0]), (void*) (&num_ss), sizeof(short));
1906 }
1907
1908 }
1909 break;
1910 case H5UINT16: {
1911 num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1912 if (num_sli < 0 || num_sli > USHRT_MAX)
1913 throw5("Attribute type is unsigned 16-bit integer, the current attribute ", attr->name, " has the value ",
1914 num_sli, ". It is overflowed. ");
1915 else {
1916 unsigned short num_uss = (unsigned short) num_sli;
1917 attr->dtype = H5UINT16;
1918 attr->value.resize(sizeof(unsigned short));
1919 memcpy(&(attr->value[0]), (void*) (&num_uss), sizeof(unsigned short));
1920 }
1921 }
1922 break;
1923 case H5INT32: {
1924 num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1925 //No need to check overflow, the number will always be in the range.
1926#if 0
1927 //if(num_sli <LONG_MIN || num_sli >LONG_MAX)
1928 // throw5("Attribute type is 32-bit integer, the current attribute ",attr->name, " has the value ",num_sli, ". It is overflowed. ");
1929#endif
1930 attr->dtype = H5INT32;
1931 attr->value.resize(sizeof(long int));
1932 memcpy(&(attr->value[0]), (void*) (&num_sli), sizeof(long int));
1933
1934 }
1935 break;
1936 case H5UINT32: {
1937 unsigned long int num_suli = strtoul(&(attr->value[0]), &pEnd, 10);
1938 // No need to check since num_suli will not be bigger than ULONG_MAX.
1939 attr->dtype = H5UINT32;
1940 attr->value.resize(sizeof(unsigned long int));
1941 memcpy(&(attr->value[0]), (void*) (&num_suli), sizeof(unsigned long int));
1942 }
1943 break;
1944 case H5FLOAT32: {
1945 float num_sf = strtof(&(attr->value[0]), nullptr);
1946 // Don't think it is necessary to check if floating-point is oveflowed for this routine. ignore it now. KY 2014-09-22
1947 attr->dtype = H5FLOAT32;
1948 attr->value.resize(sizeof(float));
1949 memcpy(&(attr->value[0]), (void*) (&num_sf), sizeof(float));
1950 }
1951 break;
1952 case H5FLOAT64: {
1953 double num_sd = strtod(&(attr->value[0]), nullptr);
1954 // Don't think it is necessary to check if floating-point is oveflowed for this routine. ignore it now. KY 2014-09-22
1955 attr->dtype = H5FLOAT64;
1956 attr->value.resize(sizeof(double));
1957 memcpy(&(attr->value[0]), (void*) (&num_sd), sizeof(double));
1958 }
1959 break;
1960
1961 default:
1962 throw4("Unsupported HDF5 datatype that the string is converted to for the attribute ", attr->name,
1963 " of the variable ", var->fullpath);
1964 } // "switch(var->dtype)"
1965
1966}
1967
1968// Change a string type attribute to the input value
1969void File::Replace_Var_Str_Attr(Var* var, const string &attr_name, const string& strvalue)
1970{
1971
1972 bool rep_attr = true;
1973 bool rem_attr = false;
1974 for (auto ira = var->attrs.begin(); ira != var->attrs.end(); ira++) {
1975 if ((*ira)->name == attr_name) {
1976 if (true == Is_Str_Attr(*ira, var->fullpath, attr_name, strvalue))
1977 rep_attr = false;
1978 else
1979 rem_attr = true;
1980 break;
1981 }
1982 }
1983
1984 // Remove the attribute if the attribute value is not strvalue
1985 if (true == rem_attr) {
1986 for (auto ira = var->attrs.begin(); ira != var->attrs.end(); ira++) {
1987 if ((*ira)->name == attr_name) {
1988 delete (*ira);
1989 var->attrs.erase(ira);
1990 break;
1991 }
1992 }
1993 }
1994
1995 // Add the attribute with strvalue
1996 if (true == rep_attr) {
1997 auto attr = new Attribute();
1998 Add_Str_Attr(attr, attr_name, strvalue);
1999 var->attrs.push_back(attr);
2000 }
2001}
2002
2003// Check if this variable if latitude,longitude.We check the three name pairs(lat,lon),(latitude,longitude),(Latitude,Longitude)
2004bool File::Is_geolatlon(const string & var_name, bool is_lat)
2005{
2006
2007 bool ret_value = false;
2008 if (true == is_lat) {
2009 string lat1 = "lat";
2010 string lat2 = "latitude";
2011 string lat3 = "Latitude";
2012
2013 if (var_name.compare(lat1) == 0 || var_name.compare(lat2) == 0 || var_name.compare(lat3) == 0) ret_value = true;
2014 }
2015
2016 else {
2017 string lon1 = "lon";
2018 string lon2 = "longitude";
2019 string lon3 = "Longitude";
2020 if (var_name.compare(lon1) == 0 || var_name.compare(lon2) == 0 || var_name.compare(lon3) == 0) ret_value = true;
2021
2022 }
2023 return ret_value;
2024}
2025
2026// Add supplementary attributes.
2027void File::Add_Supplement_Attrs(bool add_path)
2028{
2029
2030 if (false == add_path) return;
2031
2032 // Adding variable original name(origname) and full path(fullpath)
2033 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2034 auto attr = new Attribute();
2035 const string varname = (*irv)->name;
2036 const string attrname = "origname";
2037 Add_Str_Attr(attr, attrname, varname);
2038 (*irv)->attrs.push_back(attr);
2039 }
2040
2041 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2042 // Turn off the fullnamepath attribute when zero_storage_size is 0.
2043 // Use the BES key since quite a few testing cases will be affected.
2044 // KY 2020-03-23
2045 if((*irv)->zero_storage_size==false
2046 || HDF5RequestHandler::get_no_zero_size_fullnameattr() == false) {
2047 auto attr = new Attribute();
2048 const string varname = (*irv)->fullpath;
2049 const string attrname = "fullnamepath";
2050 Add_Str_Attr(attr, attrname, varname);
2051 (*irv)->attrs.push_back(attr);
2052 }
2053 }
2054
2055 // Adding group path
2056 for (auto irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
2057 // Only when this group has attributes, the original path of the group has some values. So add it.
2058 if (false == (*irg)->attrs.empty()) {
2059
2060 auto attr = new Attribute();
2061 const string varname = (*irg)->path;
2062 const string attrname = "fullnamepath";
2063 Add_Str_Attr(attr, attrname, varname);
2064 (*irg)->attrs.push_back(attr);
2065 }
2066 }
2067
2068}
2069
2070// Variable target will not be deleted, but rather its contents are replaced.
2071// We may make this as an operator = in the future.
2072// Note: the attributes can not be replaced.
2073void File::Replace_Var_Info(Var *src, Var *target)
2074{
2075
2076#if 0
2077 for_each (target->dims.begin (), target->dims.end (),
2078 delete_elem ());
2079 for_each (target->attrs.begin (), target->attrs.end (),
2080 delete_elem ());
2081#endif
2082
2083 target->newname = src->newname;
2084 target->name = src->name;
2085 target->fullpath = src->fullpath;
2086 target->rank = src->rank;
2087 target->dtype = src->dtype;
2088 target->unsupported_attr_dtype = src->unsupported_attr_dtype;
2089 target->unsupported_dspace = src->unsupported_dspace;
2090#if 0
2091 for (auto ira = target->attrs.begin();
2092 ira!=target->attrs.end(); ++ira) {
2093 delete (*ira);
2094 target->attrs.erase(ira);
2095 ira--;
2096 }
2097#endif
2098 for (auto ird = target->dims.begin(); ird != target->dims.end();) {
2099 delete (*ird);
2100 ird = target->dims.erase(ird);
2101 }
2102
2103 // Somehow attributes cannot be replaced.
2104#if 0
2105 for (vector<Attribute*>::iterator ira = src->attrs.begin();
2106 ira!=src->attrs.end(); ++ira) {
2107 Attribute* attr= new Attribute();
2108 attr->name = (*ira)->name;
2109 attr->newname = (*ira)->newname;
2110 attr->dtype =(*ira)->dtype;
2111 attr->count =(*ira)->count;
2112 attr->strsize = (*ira)->strsize;
2113 attr->fstrsize = (*ira)->fstrsize;
2114 attr->value =(*ira)->value;
2115 target->attrs.push_back(attr);
2116 }
2117#endif
2118
2119 for (auto ird = src->dims.begin(); ird != src->dims.end(); ++ird) {
2120 auto dim = new Dimension((*ird)->size);
2121 dim->name = (*ird)->name;
2122 dim->newname = (*ird)->newname;
2123 target->dims.push_back(dim);
2124 }
2125
2126}
2127
2128// Replace the attributes of target with src.
2129void File::Replace_Var_Attrs(Var *src, Var *target)
2130{
2131
2132#if 0
2133 for_each (target->dims.begin (), target->dims.end (),
2134 delete_elem ());
2135 for_each (target->attrs.begin (), target->attrs.end (),
2136 delete_elem ());
2137#endif
2138
2139 for (auto ira = target->attrs.begin(); ira != target->attrs.end();) {
2140 delete (*ira);
2141 ira = target->attrs.erase(ira);
2142 }
2143 for (auto ira = src->attrs.begin(); ira != src->attrs.end(); ++ira) {
2144 auto attr = new Attribute();
2145 attr->name = (*ira)->name;
2146 attr->newname = (*ira)->newname;
2147 attr->dtype = (*ira)->dtype;
2148 attr->count = (*ira)->count;
2149 attr->strsize = (*ira)->strsize;
2150 attr->fstrsize = (*ira)->fstrsize;
2151 attr->value = (*ira)->value;
2152 target->attrs.push_back(attr);
2153 }
2154
2155}
2156
2157// Check if a variable with a var name is under a specific group with groupname
2158// note: the variable's size at each dimension is also returned. The user must allocate the
2159// memory for the dimension sizes(an array(vector is perferred).
2160bool File::is_var_under_group(const string &varname, const string &grpname, const int var_rank,
2161 vector<size_t> & var_size)
2162{
2163
2164 bool ret_value = false;
2165 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2166
2167 if ((*irv)->rank == var_rank) {
2168 if ((*irv)->name == varname) {
2169
2170 // Obtain the variable path
2171 string var_path = HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath);
2172
2173 // Check if we find the variable under this group
2174 if (grpname == var_path) {
2175 ret_value = true;
2176 for (int i = 0; i < var_rank; i++)
2177 var_size[i] = (*irv)->getDimensions()[i]->size;
2178 break;
2179 }
2180 }
2181 } // "if((*irv)->rank == var_rank)"
2182 } // "for (vector<Var *>::iterator irv = this->vars.begin()"
2183
2184
2185 return ret_value;
2186
2187}
2189
2190 bool ret_value = false;
2191 for (auto irv = this->vars.begin();
2192 irv != this->vars.end(); ++irv) {
2193 for (auto ira = (*irv)->attrs.begin();
2194 ira != (*irv)->attrs.end(); ++ira) {
2195 if((*ira)->name =="grid_mapping") {
2196 ret_value = true;
2197 break;
2198 }
2199 }
2200 if(true == ret_value)
2201 break;
2202 }
2203
2204 return ret_value;
2205
2206}
2207
2209
2210 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2211 string attr_value;
2212 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
2213 if((*ira)->name =="grid_mapping") {
2214 Retrieve_H5_Attr_Value(*ira, (*irv)->fullpath);
2215 attr_value.resize((*ira)->value.size());
2216 copy((*ira)->value.begin(), (*ira)->value.end(), attr_value.begin());
2217 break;
2218 }
2219
2220 }
2221 if(attr_value.find('/') ==string::npos){
2222 string new_name = Check_Grid_Mapping_VarName(attr_value,(*irv)->fullpath);
2223 if(new_name != "")
2224 Replace_Var_Str_Attr((*irv),"grid_mapping",new_name);
2225
2226 }
2227 else {
2228 string new_name = Check_Grid_Mapping_FullPath(attr_value);
2229 //Using new_name as the attribute value
2230 if(new_name != "")
2231 Replace_Var_Str_Attr((*irv),"grid_mapping",new_name);
2232 }
2233 }
2234
2235}
2236
2237string File::Check_Grid_Mapping_VarName(const string & a_value,const string & var_fpath) {
2238
2239 string var_path = HDF5CFUtil::obtain_string_before_lastslash(var_fpath);
2240 string gmap_new_name;
2241 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2242 if((*irv)->name == a_value){
2243 if(var_path == HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath)) {
2244 gmap_new_name = (*irv)->newname;
2245 break;
2246 }
2247 }
2248 }
2249 return gmap_new_name;
2250}
2251
2252
2253string File::Check_Grid_Mapping_FullPath(const string & a_value) {
2254
2255 string gmap_new_name;
2256 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2257 if((*irv)->fullpath == a_value){
2258 gmap_new_name = (*irv)->newname;
2259 break;
2260 }
2261 }
2262
2263 return gmap_new_name;
2264}
2265
2266void File::remove_netCDF_internal_attributes(bool include_attr) {
2267
2268 if(true == include_attr) {
2269 for (auto irv = this->vars.begin();
2270 irv != this->vars.end(); ++irv) {
2271 bool var_has_dimscale = false;
2272
2273 for(auto ira = (*irv)->attrs.begin();
2274 ira != (*irv)->attrs.end();) {
2275 if((*ira)->name == "CLASS") {
2276 string class_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
2277
2278 // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
2279 // "DIMENSION_SCALE", which is 15.
2280 if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
2281 delete(*ira);
2282 ira = (*irv)->attrs.erase(ira);
2283 var_has_dimscale = true;
2284
2285 }
2286#if 0
2287 else if(1) {// Add a BES key,also delete
2288
2289 }
2290#endif
2291 else {
2292 ++ira;
2293 }
2294 }
2295#if 0
2296 else if((*ira)->name == "NAME") {// Add a BES Key
2297 string name_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
2298 if( 0 == name_value.compare(0,(*irv)->name.size(),(*irv)->name)) {
2299 delete(*ira);
2300 ira =(*irv)->attrs.erase(ira);
2301 }
2302 else {
2303 string netcdf_dim_mark= "This is a netCDF dimension but not a netCDF variable";
2304 if( 0 == name_value.compare(0,netcdf_dim_mark.size(),netcdf_dim_mark)) {
2305 delete(*ira);
2306 ira =(*irv)->attrs.erase(ira);
2307 }
2308 else {
2309 ++ira;
2310 }
2311 }
2312
2313 }
2314#endif
2315 else if((*ira)->name == "_Netcdf4Dimid") {
2316 delete(*ira);
2317 ira =(*irv)->attrs.erase(ira);
2318 }
2319 else if((*ira)->name == "_Netcdf4Coordinates") {
2320 delete(*ira);
2321 ira =(*irv)->attrs.erase(ira);
2322 }
2323#if 0
2324 else if((*ira)->name == "_nc3_strict") {
2325 delete((*ira));
2326 ira =(*irv)->attrs.erase(ira);
2327 }
2328#endif
2329 else {
2330 ++ira;
2331 }
2332 }
2333
2334 if(true == var_has_dimscale) {
2335 for(auto ira = (*irv)->attrs.begin();
2336 ira != (*irv)->attrs.end();++ira) {
2337 if((*ira)->name == "NAME") {// Add a BES Key
2338 delete(*ira);
2339 ira =(*irv)->attrs.erase(ira);
2340 break;
2341 }
2342 }
2343 }
2344 }
2345 }
2346
2347}
2348// Add ignored page header info. Mainly a helper message.
2349void File::add_ignored_info_page_header()
2350{
2351 ignored_msg =
2352 " \n This page is for HDF5 CF hyrax data providers or distributors to check if any HDF5 object or attribute information are ignored during the mapping. \n\n";
2353}
2354
2355// Add ignored object header info. Mainly a helper message.
2356void File::add_ignored_info_obj_header()
2357{
2358
2359 ignored_msg += " Some HDF5 objects or the object information are ignored when mapping to DAP2 by the HDF5 OPeNDAP";
2360 ignored_msg += " handler due to the restrictions of DAP2, CF conventions or CF tools.";
2361 ignored_msg += " Please use HDF5 tools(h5dump or HDFView) to check carefully and make sure that these objects";
2362 ignored_msg +=
2363 " are OK to ignore for your service. For questions or requests to find a way to handle the ignored objects, please";
2364 ignored_msg += " contact the HDF5 OPeNDAP handler developer or send an email to help@hdfgroup.org.\n";
2365
2366 ignored_msg += " \n In general, ignored HDF5 objects include HDF5 soft links, external links and named datatype.\n";
2367 ignored_msg +=
2368 " \n The HDF5 datasets(variables in the CF term) and attributes that have the following datatypes are ignored: \n";
2369 ignored_msg +=
2370 " Signed and unsigned 64-bit integers, HDF5 compound, HDF5 variable length(excluding variable length string),";
2371 ignored_msg += " HDF5 reference, HDF5 enum, HDF5 opaque , HDF5 bitfield, HDF5 Array and HDF5 Time datatypes.\n";
2372
2373 ignored_msg +=
2374 " \n The HDF5 datasets(variables in the CF term) and attributes associated with the following dimensions are ignored: \n";
2375 ignored_msg += " 1) variables that have HDF5 NULL dataspace(H5S_NULL)(rarely occurred)\n";
2376 ignored_msg += " 2) attributes that have any zero size dimensions(not reported due to extreme rarity and non-trivial coding)\n\n";
2377
2378}
2379
2380// Add the ignored links information.Mainly a helper message.
2381void File::add_ignored_info_links_header()
2382{
2383
2384 if (false == this->have_ignored) {
2385 add_ignored_info_obj_header();
2386 have_ignored = true;
2387 }
2388 // Add ignored datatype header.
2389 string lh_msg = "******WARNING******\n";
2390 lh_msg += "IGNORED soft links or external links are: ";
2391 if (ignored_msg.rfind(lh_msg) == string::npos) ignored_msg += lh_msg + "\n";
2392
2393}
2394
2395// Leave the code for the time being.
2396#if 0
2397void
2398File:: add_ignored_info_obj_dtype_header() {
2399
2400 // Add ignored datatype header.
2401 ignored_msg += " \n Variables and attributes ignored due to the unsupported datatypes. \n";
2402 ignored_msg += " In general, the unsupported datatypes include: \n";
2403 ignored_msg += " Signed and unsigned 64-bit integers, HDF5 compound, HDF5 variable length(excluding variable length string),";
2404 ignored_msg += " HDF5 reference, HDF5 enum, HDF5 opaque , HDF5 bitfield, HDF5 Array and HDF5 Time datatypes.\n";
2405
2406}
2407
2408void
2409File:: add_ignored_info_obj_dspace_header() {
2410
2411 // Add ignored dataspace header.
2412 ignored_msg += " \n Variables and attributes ignored due to the unsupported dimensions. \n";
2413 ignored_msg += " In general, the unsupported dimensions include: \n";
2414 ignored_msg += " 1) variables that have HDF5 NULL dataspace(H5S_NULL)(rarely occurred)\n";
2415 ignored_msg += " 2) variables that have any zero size dimensions\n";
2416
2417}
2418#endif
2419
2420// Add the ignored link info.
2421void File::add_ignored_info_links(const string & link_path)
2422{
2423 if (ignored_msg.find("Link paths: ") == string::npos)
2424 ignored_msg += " Link paths: " + link_path;
2425 else
2426 ignored_msg += " " + link_path;
2427}
2428
2429// Add the ignored name datatype info.
2430void File::add_ignored_info_namedtypes(const string& grp_name, const string& named_dtype_name)
2431{
2432
2433 if (false == this->have_ignored) {
2434 add_ignored_info_obj_header();
2435 have_ignored = true;
2436 }
2437
2438 string ignored_HDF5_named_dtype_hdr = "\n******WARNING******";
2439 ignored_HDF5_named_dtype_hdr += "\n IGNORED HDF5 named datatype objects:\n";
2440 string ignored_HDF5_named_dtype_msg = " Group name: " + grp_name + " HDF5 named datatype name: " + named_dtype_name.substr(0,named_dtype_name.size()-1)
2441 + "\n";
2442 if (ignored_msg.find(ignored_HDF5_named_dtype_hdr) == string::npos)
2443 ignored_msg += ignored_HDF5_named_dtype_hdr + ignored_HDF5_named_dtype_msg;
2444 else
2445 ignored_msg += ignored_HDF5_named_dtype_msg;
2446
2447}
2448
2449// Add the ignored attribute information. When is_grp is true, the ignored group attribute names are added.
2450// Otherwise, the ignored dataset attribute names are added.
2451void File::add_ignored_info_attrs(bool is_grp, const string & obj_path, const string & attr_name)
2452{
2453
2454 if (false == this->have_ignored) {
2455 add_ignored_info_obj_header();
2456 have_ignored = true;
2457 }
2458
2459
2460 string ignored_warning_str = "\n******WARNING******";
2461 string ignored_HDF5_grp_hdr = ignored_warning_str + "\n Ignored attributes under root and groups:\n";
2462 string ignored_HDF5_grp_msg = " Group path: " + obj_path + " Attribute names: " + attr_name + "\n";
2463 string ignored_HDF5_var_hdr = ignored_warning_str + "\n Ignored attributes for variables:\n";
2464 string ignored_HDF5_var_msg = " Variable path: " + obj_path + " Attribute names: " + attr_name + "\n";
2465
2466
2467 if (true == is_grp) {
2468 if (ignored_msg.find(ignored_HDF5_grp_hdr) == string::npos)
2469 ignored_msg += ignored_HDF5_grp_hdr + ignored_HDF5_grp_msg;
2470 else
2471 ignored_msg += ignored_HDF5_grp_msg;
2472 }
2473 else {
2474 if (ignored_msg.find(ignored_HDF5_var_hdr) == string::npos)
2475 ignored_msg += ignored_HDF5_var_hdr + ignored_HDF5_var_msg;
2476 else
2477 ignored_msg += ignored_HDF5_var_msg;
2478 }
2479
2480}
2481
2482//Ignored object information. When is_dim_related is true, ignored data space info. is present.
2483//When is_dim_related is false, ignored data type info. is present.
2484void File::add_ignored_info_objs(bool is_dim_related, const string & obj_path)
2485{
2486
2487 if (false == this->have_ignored) {
2488 add_ignored_info_obj_header();
2489 have_ignored = true;
2490 }
2491
2492 string ignored_warning_str = "\n******WARNING******";
2493 string ignored_HDF5_dtype_var_hdr = ignored_warning_str + "\n IGNORED variables due to unsupported datatypes:\n";
2494 string ignored_HDF5_dspace_var_hdr = ignored_warning_str + "\n IGNORED variables due to unsupported dimensions:\n";
2495 string ignored_HDF5_var_msg = " Variable path: " + obj_path + "\n";
2496
2497 if (true == is_dim_related) {
2498 if (ignored_msg.find(ignored_HDF5_dspace_var_hdr) == string::npos)
2499 ignored_msg += ignored_HDF5_dspace_var_hdr + ignored_HDF5_var_msg;
2500 else
2501 ignored_msg += ignored_HDF5_var_msg;
2502
2503 }
2504 else {
2505 if (ignored_msg.find(ignored_HDF5_dtype_var_hdr) == string::npos)
2506 ignored_msg += ignored_HDF5_dtype_var_hdr + ignored_HDF5_var_msg;
2507 else
2508 ignored_msg += ignored_HDF5_var_msg;
2509 }
2510
2511}
2512
2513// No ignored info.
2514void File::add_no_ignored_info()
2515{
2516
2517 ignored_msg += "There are no ignored HDF5 objects or attributes.";
2518
2519}
2520
2521// This function should only be used when the HDF5 file is following the netCDF data model.
2522// Check if we should not report the Dimension scale related attributes as ignored.
2523bool File::ignored_dimscale_ref_list(Var *var)
2524{
2525
2526 bool ignored_dimscale = true;
2527 // Only when "General_Product == this->product_type && GENERAL_DIMSCALE== this->gproduct_pattern)"
2528
2529 bool has_dimscale = false;
2530 bool has_reference_list = false;
2531 for (auto ira = var->attrs.begin(); ira != var->attrs.end(); ira++) {
2532 if ((*ira)->name == "REFERENCE_LIST" && false == HDF5CFUtil::cf_strict_support_type((*ira)->getType(),_is_dap4))
2533 has_reference_list = true;
2534 if ((*ira)->name == "CLASS") {
2535 Retrieve_H5_Attr_Value(*ira, var->fullpath);
2536 string class_value;
2537 class_value.resize((*ira)->value.size());
2538 copy((*ira)->value.begin(), (*ira)->value.end(), class_value.begin());
2539
2540 // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
2541 // "DIMENSION_SCALE", which is 15.
2542 if (0 == class_value.compare(0, 15, "DIMENSION_SCALE")) {
2543 has_dimscale = true;
2544 }
2545 }
2546
2547 if (true == has_dimscale && true == has_reference_list) {
2548 ignored_dimscale = false;
2549 break;
2550 }
2551
2552 }
2553 //}
2554 return ignored_dimscale;
2555}
2556
2557// Check if the long string can should be dropped from a dataset or an attribute. Users can set up a BES key to turn it off or on.
2558bool File::Check_DropLongStr(const Var *var, const Attribute * attr)
2559{
2560
2561 bool drop_longstr = false;
2562 if (nullptr == attr) {
2563 if (H5FSTRING == var->dtype || H5VSTRING == var->dtype) {
2564 try {
2565 drop_longstr = Check_VarDropLongStr(var->fullpath, var->dims, var->dtype);
2566 }
2567 catch (...) {
2568 throw1("Check_VarDropLongStr fails ");
2569 }
2570 }
2571 }
2572 // No limitation for the attributes. KY 2018-02-26
2573#if 0
2574 else {
2575 if (H5FSTRING == attr->dtype || H5VSTRING == attr->dtype) {
2576 if (attr->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
2577 drop_longstr = true;
2578 }
2579 }
2580
2581 }
2582#endif
2583 return drop_longstr;
2584}
2585
2586// Check if a long string dataset should be dropped. Users can turn on a BES key not to drop the long string.
2587// However, the Java clients may not access.
2588//
2589bool File::Check_VarDropLongStr(const string & varpath, const vector<Dimension *>& dims, H5DataType dtype) const
2590
2591{
2592
2593 bool drop_longstr = false;
2594
2595 hid_t dset_id = H5Dopen2(this->fileid, varpath.c_str(), H5P_DEFAULT);
2596 if (dset_id < 0)
2597 throw2("Cannot open the dataset ", varpath);
2598
2599 hid_t dtype_id = -1;
2600 if ((dtype_id = H5Dget_type(dset_id)) < 0) {
2601 H5Dclose(dset_id);
2602 throw2("Cannot obtain the datatype of the dataset ", varpath);
2603 }
2604
2605 size_t ty_size = H5Tget_size(dtype_id);
2606 if (ty_size == 0) {
2607 H5Tclose(dtype_id);
2608 H5Dclose(dset_id);
2609 throw2("Cannot obtain the datatype size of the dataset ", varpath);
2610 }
2611
2612 if (H5FSTRING == dtype) { // Fixed-size, just check the number of elements.
2613 if (ty_size > NC_JAVA_STR_SIZE_LIMIT) drop_longstr = true;
2614 }
2615 else if (H5VSTRING == dtype) {
2616
2617 unsigned long long total_elms = 1;
2618 if (dims.size() != 0) {
2619 for (unsigned int i = 0; i < dims.size(); i++)
2620 total_elms = total_elms * ((dims[i])->size);
2621 }
2622 vector<char> strval;
2623 strval.resize(total_elms * ty_size);
2624 hid_t read_ret = H5Dread(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, (void*) strval.data());
2625 if (read_ret < 0) {
2626 H5Tclose(dtype_id);
2627 H5Dclose(dset_id);
2628 throw2("Cannot read the data of the dataset ", varpath);
2629 }
2630
2631 vector<string> finstrval;
2632 finstrval.resize(total_elms);
2633 char*temp_bp = strval.data();
2634 char*onestring = nullptr;
2635 for (unsigned long long i = 0; i < total_elms; i++) {
2636 onestring = *(char**) temp_bp;
2637 if (onestring != nullptr) {
2638 finstrval[i] = string(onestring);
2639 if(finstrval[i].size()>NC_JAVA_STR_SIZE_LIMIT) {
2640 drop_longstr = true;
2641 break;
2642 }
2643 }
2644 temp_bp += ty_size;
2645 }
2646
2647 if (false == strval.empty()) {
2648 herr_t ret_vlen_claim;
2649 hid_t dspace_id = H5Dget_space(dset_id);
2650 if (dspace_id < 0) {
2651 H5Tclose(dtype_id);
2652 H5Dclose(dset_id);
2653 throw2("Cannot obtain the dataspace id.", varpath);
2654 }
2655 ret_vlen_claim = H5Dvlen_reclaim(dtype_id, dspace_id, H5P_DEFAULT, (void*) strval.data());
2656 if (ret_vlen_claim < 0) {
2657 H5Tclose(dtype_id);
2658 H5Sclose(dspace_id);
2659 H5Dclose(dset_id);
2660 throw2("Cannot reclaim the vlen space ", varpath);
2661 }
2662 if (H5Sclose(dspace_id) < 0) {
2663 H5Tclose(dtype_id);
2664 H5Dclose(dset_id);
2665 throw2("Cannot close the HDF5 data space.", varpath);
2666 }
2667 }
2668 }
2669 if (H5Tclose(dtype_id) < 0) {
2670 H5Dclose(dset_id);
2671 throw2("Cannot close the HDF5 data type.", varpath);
2672 }
2673 if (H5Dclose(dset_id) < 0)
2674 throw2("Cannot close the HDF5 data type.", varpath);
2675
2676 return drop_longstr;
2677}
2678#if 0
2679bool File::Check_VarDropLongStr(const string & varpath, const vector<Dimension *>& dims, H5DataType dtype)
2680
2681{
2682
2683 bool drop_longstr = false;
2684
2685 unsigned long long total_elms = 1;
2686 if (dims.size() != 0) {
2687 for (unsigned int i = 0; i < dims.size(); i++)
2688 total_elms = total_elms * ((dims[i])->size);
2689 }
2690
2691 if (total_elms > NC_JAVA_STR_SIZE_LIMIT)
2692 drop_longstr = true;
2693
2694 else { // We need to check both fixed-size and variable-length strings.
2695
2696 hid_t dset_id = H5Dopen2(this->fileid, varpath.c_str(), H5P_DEFAULT);
2697 if (dset_id < 0)
2698 throw2("Cannot open the dataset ", varpath);
2699
2700 hid_t dtype_id = -1;
2701 if ((dtype_id = H5Dget_type(dset_id)) < 0) {
2702 H5Dclose(dset_id);
2703 throw2("Cannot obtain the datatype of the dataset ", varpath);
2704 }
2705
2706 size_t ty_size = H5Tget_size(dtype_id);
2707 if (ty_size == 0) {
2708 H5Tclose(dtype_id);
2709 H5Dclose(dset_id);
2710 throw2("Cannot obtain the datatype size of the dataset ", varpath);
2711 }
2712
2713 if (H5FSTRING == dtype) { // Fixed-size, just check the number of elements.
2714 if ((ty_size * total_elms) > NC_JAVA_STR_SIZE_LIMIT) drop_longstr = true;
2715 }
2716 else if (H5VSTRING == dtype) {
2717
2718 vector<char> strval;
2719 strval.resize(total_elms * ty_size);
2720 hid_t read_ret = H5Dread(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, (void*) strval.data());
2721 if (read_ret < 0) {
2722 H5Tclose(dtype_id);
2723 H5Dclose(dset_id);
2724 throw2("Cannot read the data of the dataset ", varpath);
2725 }
2726
2727 vector<string> finstrval;
2728 finstrval.resize(total_elms);
2729 char*temp_bp = strval.data();
2730 char*onestring = nullptr;
2731 for (unsigned long long i = 0; i < total_elms; i++) {
2732 onestring = *(char**) temp_bp;
2733 if (onestring != nullptr)
2734 finstrval[i] = string(onestring);
2735 else
2736 // We will add a NULL if onestring is NULL.
2737 finstrval[i] = "";
2738 temp_bp += ty_size;
2739 }
2740
2741 if (false == strval.empty()) {
2742 herr_t ret_vlen_claim;
2743 hid_t dspace_id = H5Dget_space(dset_id);
2744 if (dspace_id < 0) {
2745 H5Tclose(dtype_id);
2746 H5Dclose(dset_id);
2747 throw2("Cannot obtain the dataspace id.", varpath);
2748 }
2749 ret_vlen_claim = H5Dvlen_reclaim(dtype_id, dspace_id, H5P_DEFAULT, (void*) strval.data());
2750 if (ret_vlen_claim < 0) {
2751 H5Tclose(dtype_id);
2752 H5Sclose(dspace_id);
2753 H5Dclose(dset_id);
2754 throw2("Cannot reclaim the vlen space ", varpath);
2755 }
2756 if (H5Sclose(dspace_id) < 0) {
2757 H5Tclose(dtype_id);
2758 H5Dclose(dset_id);
2759 throw2("Cannot close the HDF5 data space.", varpath);
2760 }
2761 }
2762 unsigned long long total_str_size = 0;
2763 for (unsigned long long i = 0; i < total_elms; i++) {
2764 total_str_size += finstrval[i].size();
2765 if (total_str_size > NC_JAVA_STR_SIZE_LIMIT) {
2766 drop_longstr = true;
2767 break;
2768 }
2769 }
2770 }
2771 if (H5Tclose(dtype_id) < 0) {
2772 H5Dclose(dset_id);
2773 throw2("Cannot close the HDF5 data type.", varpath);
2774 }
2775 if (H5Dclose(dset_id) < 0)
2776 throw2("Cannot close the HDF5 data type.", varpath);
2777 }
2778 return drop_longstr;
2779}
2780#endif
2781
2782
2783// Provide if the long string is dropped.
2784void File::add_ignored_grp_longstr_info(const string& grp_path, const string & attr_name)
2785{
2786
2787 ignored_msg += "The HDF5 group: " + grp_path + " has an empty-set string attribute: " + attr_name + "\n";
2788
2789 return;
2790}
2791
2792// Provide if the long variable string is dropped.
2793void File::add_ignored_var_longstr_info(const Var *var, const Attribute *attr)
2794{
2795
2796 if (nullptr == attr)
2797 ignored_msg += "String variable: " + var->fullpath + " value is set to empty.\n";
2798 else {
2799 ignored_msg += "The variable: " + var->fullpath + " has an empty-set string attribute: " + attr->name + "\n";
2800
2801 }
2802 return;
2803}
2804
2805// The warnings of the drop of the long string header
2806void File::add_ignored_droplongstr_hdr()
2807{
2808
2809 if (false == this->have_ignored) this->have_ignored = true;
2810 string hdr = "\n\n The values of the following string variables ";
2811 hdr += " are set to empty because at least one string size in this variable exceeds netCDF Java string limit(32767 bytes).\n";
2812 hdr += "To obtain the values, change the BES key H5.EnableDropLongString=true at the handler BES";
2813 hdr += " configuration file(h5.conf)\nto H5.EnableDropLongString=false.\n\n";
2814
2815 if (ignored_msg.rfind(hdr) == string::npos) ignored_msg += hdr;
2816
2817}
2818
2819// Sometimes, we need to release the temporary added resources.
2820void File::release_standalone_var_vector(vector<Var*>&temp_vars)
2821{
2822
2823 for (auto i = temp_vars.begin(); i != temp_vars.end();) {
2824 delete (*i);
2825 i = temp_vars.erase(i);
2826 }
2827
2828}
This class specifies the core engineering of mapping HDF5 to DAP by following CF.
include the entry functions to execute the handlers
This class represents one attribute.
Definition: HDF5CF.h:189
This class repersents one dimension of an HDF5 dataset(variable).
Definition: HDF5CF.h:145
std::vector< Group * > groups
Non-root group vectors.
Definition: HDF5CF.h:795
virtual void Retrieve_H5_Var_Attr_Values(Var *var)
Retrieve attribute values for a variable.
Definition: HDF5CF.cc:746
virtual void Handle_Unsupported_Dspace(bool)
Handle unsupported HDF5 dataspaces for datasets.
Definition: HDF5CF.cc:1273
std::map< hsize_t, std::string > dimsize_to_fakedimname
Handle added dimension names.
Definition: HDF5CF.h:812
virtual void Handle_Grid_Mapping_Vars()
Handle Grid Mapping Vars.
Definition: HDF5CF.cc:2208
virtual void Handle_Unsupported_Others(bool)
Handle other unmapped objects/attributes.
Definition: HDF5CF.cc:1320
virtual void Retrieve_H5_Supported_Attr_Values()
Retrieve attribute values for the supported HDF5 datatypes.
Definition: HDF5CF.cc:727
std::vector< Var * > vars
Var vectors.
Definition: HDF5CF.h:789
virtual void Add_Supplement_Attrs(bool)
Add supplemental attributes such as fullpath and original name.
Definition: HDF5CF.cc:2027
virtual void Retrieve_H5_Info(const char *path, hid_t file_id, bool)
Definition: HDF5CF.cc:168
std::vector< Attribute * > root_attrs
Root attribute vectors.
Definition: HDF5CF.h:792
virtual void Handle_Unsupported_Dtype(bool)
Handle unsupported HDF5 datatypes.
Definition: HDF5CF.cc:916
virtual void Flatten_Obj_Name(bool)
Flatten the object name.
Definition: HDF5CF.cc:1370
virtual bool Have_Grid_Mapping_Attrs()
Check if having Grid Mapping Attrs.
Definition: HDF5CF.cc:2188
This class represents an HDF5 group. The group will be flattened according to the CF conventions.
Definition: HDF5CF.h:525
This class represents one HDF5 dataset(CF variable)
Definition: HDF5CF.h:256
Helper functions for generating DAS attributes and a function to check BES Key.
static H5DataType H5type_to_H5DAPtype(hid_t h5_type_id)
Map HDF5 Datatype to the intermediate H5DAPtype for the future use.
Definition: HDF5CFUtil.cc:54
static std::string trim_string(hid_t dtypeid, const std::string &s, int num_sect, size_t section_size, std::vector< size_t > &sect_newsize)
Definition: HDF5CFUtil.cc:235