bes Updated for version 3.20.13
h5gmcfdap.cc
Go to the documentation of this file.
1// This file is part of hdf5_handler: an HDF5 file handler for the OPeNDAP
2// data server.
3
4// Copyright (c) 2011-2016 The HDF Group, Inc. and OPeNDAP, Inc.
5//
6// This is free software; you can redistribute it and/or modify it under the
7// terms of the GNU Lesser General Public License as published by the Free
8// Software Foundation; either version 2.1 of the License, or (at your
9// option) any later version.
10//
11// This software is distributed in the hope that it will be useful, but
12// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14// License for more details.
15//
16// You should have received a copy of the GNU Lesser General Public
17// License along with this library; if not, write to the Free Software
18// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19//
20// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
21// You can contact The HDF Group, Inc. at 1800 South Oak Street,
22// Suite 203, Champaign, IL 61820
23
32
33#include <sys/types.h>
34#include <sys/stat.h>
35#include <fcntl.h>
36#include <unistd.h>
37#include <iostream>
38#include <sstream>
39
40#include <BESDebug.h>
41#include <libdap/InternalErr.h>
42
43#include "HDF5RequestHandler.h"
44#include "h5cfdaputil.h"
45#include "h5gmcfdap.h"
46#include "HDF5CFInt8.h"
47#include "HDF5CFByte.h"
48#include "HDF5CFUInt16.h"
49#include "HDF5CFInt16.h"
50#include "HDF5CFUInt32.h"
51#include "HDF5CFInt32.h"
52#include "HDF5CFInt64.h"
53#include "HDF5CFUInt64.h"
54#include "HDF5CFFloat32.h"
55#include "HDF5CFFloat64.h"
56#include "HDF5CFStr.h"
57#include "HDF5CFArray.h"
58#include "HDF5GMCFMissLLArray.h"
62#include "HDF5GMSPCFArray.h"
63
64using namespace std;
65using namespace libdap;
66using namespace HDF5CF;
67
68// Map general HDF5 products to DAP DDS
69void map_gmh5_cfdds(DDS &dds, hid_t file_id, const string& filename){
70
71 BESDEBUG("h5","Coming to GM products DDS mapping function map_gmh5_cfdds() "<<endl);
72
73 H5GCFProduct product_type = check_product(file_id);
74
75 GMPattern gproduct_pattern = OTHERGMS;
76
77 GMFile * f = nullptr;
78
79 try {
80 f = new GMFile(filename.c_str(),file_id,product_type,gproduct_pattern);
81 }
82 catch(...) {
83 throw InternalErr(__FILE__,__LINE__,"Cannot allocate memory for GMFile ");
84 }
85 // Generally don't need to handle attributes when handling DDS.
86 bool include_attr = false;
87 try {
88 // Retrieve all HDF5 info(Not the values)
89 f->Retrieve_H5_Info(filename.c_str(),file_id,include_attr);
90
91 // Update product type
92 // Newer version of a product may have different layout and the
93 // product type needs to be changed to reflect it. We also want
94 // to support the older version in case people still use them.
95 // This routine will check if newer layout can be applied. If yes,
96 // update the product type.
98
100
101 // Need to add dimension names.
102 f->Add_Dim_Name();
103
104 // Handle coordinate variables
105 f->Handle_CVar();
106#if 0
107 // We need to retrieve coordinate variable attributes for memory cache use.
108 //f->Retrieve_H5_CVar_Supported_Attr_Values();
109 //if((HDF5RequestHandler::get_lrdata_mem_cache() != nullptr) ||
110 // (HDF5RequestHandler::get_srdata_mem_cache() != nullptr)){
111 // f->Retrieve_H5_Supported_Attr_Values();
112#endif
113
114
115 // Handle special variables
116 f->Handle_SpVar();
117
118 // When cv memory cache is on, the unit attributes are needed to
119 // distinguish whether this is lat/lon. Generally, memory cache
120 // is not used. This snipnet will not be accessed.
121 if((HDF5RequestHandler::get_lrdata_mem_cache() != nullptr) ||
122 (HDF5RequestHandler::get_srdata_mem_cache() != nullptr)){
123
124 // Handle unsupported datatypes including the attributes
126
127 // Handle unsupported dataspaces including the attributes
129
130 // We need to retrieve coordinate variable attributes for memory cache use.
132
133 }
134 else {
135
136 // Handle unsupported datatypes
137 f->Handle_Unsupported_Dtype(include_attr);
138
139 // Handle unsupported dataspaces
140 f->Handle_Unsupported_Dspace(include_attr);
141
142 }
143
144 // Need to handle the "coordinate" attributes when memory cache is turned on.
145 if((HDF5RequestHandler::get_lrdata_mem_cache() != nullptr) ||
146 (HDF5RequestHandler::get_srdata_mem_cache() != nullptr))
147 f->Add_Supplement_Attrs(HDF5RequestHandler::get_add_path_attrs());
148
149 // Adjust object names(may remove redundant paths)
150 f->Adjust_Obj_Name();
151
152 // Flatten the object names
153 f->Flatten_Obj_Name(include_attr);
154
155 // Handle Object name clashings
156 // Only when the check_nameclashing key is turned on or
157 // general product.
158 if(General_Product == product_type ||
159 true == HDF5RequestHandler::get_check_name_clashing())
160 f->Handle_Obj_NameClashing(include_attr);
161
162 // Adjust Dimension name
163 f->Adjust_Dim_Name();
164 if(General_Product == product_type ||
165 true == HDF5RequestHandler::get_check_name_clashing())
167
168 f->Handle_Hybrid_EOS5();
169 if(true == f->Have_Grid_Mapping_Attrs())
171 // Need to handle the "coordinate" attributes when memory cache is turned on.
172 if((HDF5RequestHandler::get_lrdata_mem_cache() != nullptr) ||
173 (HDF5RequestHandler::get_srdata_mem_cache() != nullptr))
174 f->Handle_Coor_Attr();
175
178 }
179 catch (HDF5CF::Exception &e){
180 if (f != nullptr)
181 delete f;
182 throw InternalErr(e.what());
183 }
184
185 // generate DDS.
186 try {
187 gen_gmh5_cfdds(dds,f);
188 }
189 catch(...) {
190 if (f != nullptr)
191 delete f;
192 throw;
193 }
194
195 if (f != nullptr)
196 delete f;
197}
198
199// Map general HDF5 products to DAP DAS
200void map_gmh5_cfdas(DAS &das, hid_t file_id, const string& filename){
201
202 BESDEBUG("h5","Coming to GM products DAS mapping function map_gmh5_cfdas() "<<endl);
203
204 H5GCFProduct product_type = check_product(file_id);
205 GMPattern gproduct_pattern = OTHERGMS;
206
207 GMFile *f = nullptr;
208
209 try {
210 f = new GMFile(filename.c_str(),file_id,product_type,gproduct_pattern);
211 }
212 catch(...) {
213 throw InternalErr(__FILE__,__LINE__,"Cannot allocate memory for GMFile ");
214 }
215
216 bool include_attr = true;
217 try {
218 f->Retrieve_H5_Info(filename.c_str(),file_id,include_attr);
219
220 // Update product type(check comments of map_gmh5_cfdds)
222
224
225 f->Add_Dim_Name();
226 f->Handle_CVar();
227 f->Handle_SpVar();
228 f->Handle_Unsupported_Dtype(include_attr);
229
230 // Remove unsupported dataspace
231 f->Handle_Unsupported_Dspace(include_attr);
232
233 // Need to retrieve the attribute values to feed DAS
235
236 // Handle other unsupported objects,
237 // currently it mainly generates the info. for the
238 // unsupported objects other than datatype, dataspace,links and named datatype
239 // One area is maybe very long string. So we retrieve the attribute
240 // values before calling this function.
241 f->Handle_Unsupported_Others(include_attr);
242
243
244 // Need to add original variable name and path
245 // and other special attributes
246 // Can be turned on/off by using the check_path_attrs keys.
247 f->Add_Supplement_Attrs(HDF5RequestHandler::get_add_path_attrs());
248 f->Adjust_Obj_Name();
249 f->Flatten_Obj_Name(include_attr);
250 if(General_Product == product_type ||
251 true == HDF5RequestHandler::get_check_name_clashing())
252 f->Handle_Obj_NameClashing(include_attr);
253 if(f->HaveUnlimitedDim() == true)
254 f->Adjust_Dim_Name();
255 // Handle the "coordinate" attributes.
256 f->Handle_Coor_Attr();
257
258 f->Handle_Hybrid_EOS5();
259 if(true == f->Have_Grid_Mapping_Attrs())
261
263
265
266 if(true == HDF5RequestHandler::get_enable_coord_attr_add_path())
268 }
269 catch (HDF5CF::Exception &e){
270 if (f!= nullptr)
271 delete f;
272 throw InternalErr(e.what());
273 }
274
275 // Generate the DAS attributes.
276 try {
277 gen_gmh5_cfdas(das,f);
278 }
279 catch (...) {
280 if (f!= nullptr)
281 delete f;
282 throw;
283
284 }
285
286 if (f != nullptr)
287 delete f;
288}
289
290
291void map_gmh5_cfdmr(D4Group *d4_root, hid_t file_id, const string& filename){
292
293 BESDEBUG("h5","Coming to GM products DMR mapping function map_gmh5_cfdmr() "<<endl);
294
295 H5GCFProduct product_type = check_product(file_id);
296
297 GMPattern gproduct_pattern = OTHERGMS;
298
299 GMFile * f = nullptr;
300
301 try {
302 f = new GMFile(filename.c_str(),file_id,product_type,gproduct_pattern);
303 }
304 catch(...) {
305 throw InternalErr(__FILE__,__LINE__,"Cannot allocate memory for GMFile ");
306 }
307
308 // Both variables and attributes are in DMR.
309 bool include_attr = true;
310 try {
311
312 // Set the is_dap4 flag be true.
313 f->setDap4(true);
314
315 // Retrieve all HDF5 info(Not the values)
316 f->Retrieve_H5_Info(filename.c_str(),file_id,include_attr);
317
318 // Update product type
319 // Newer version of a product may have different layout and the
320 // product type needs to be changed to reflect it. We also want
321 // to support the older version in case people still use them.
322 // This routine will check if newer layout can be applied. If yes,
323 // update the product type.
325
327
328 // Need to add dimension names.
329 f->Add_Dim_Name();
330
331 // Handle coordinate variables
332 f->Handle_CVar();
333#if 0
334 // We need to retrieve coordinate variable attributes for memory cache use.
335 //f->Retrieve_H5_CVar_Supported_Attr_Values();
336 //if((HDF5RequestHandler::get_lrdata_mem_cache() != nullptr) ||
337 // (HDF5RequestHandler::get_srdata_mem_cache() != nullptr)){
338 // f->Retrieve_H5_Supported_Attr_Values();
339#endif
340
341
342 // Handle special variables
343 f->Handle_SpVar();
344
345 // Handle unsupported datatypes including the attributes
347
348 // Handle unsupported dataspaces including the attributes
350
351 // We need to retrieve variable attributes.
353
354 // Include handling internal netCDF-4 attributes
355 f->Handle_Unsupported_Others(include_attr);
356
357 // Need to handle the "coordinate" attributes
358 f->Add_Supplement_Attrs(HDF5RequestHandler::get_add_path_attrs());
359
360 // Adjust object names(may remove redundant paths)
361 f->Adjust_Obj_Name();
362
363 // Flatten the object names
364 f->Flatten_Obj_Name(include_attr);
365
366 // Handle Object name clashings
367 // Only when the check_nameclashing key is turned on or
368 // general product.
369 if(General_Product == product_type ||
370 true == HDF5RequestHandler::get_check_name_clashing())
371 f->Handle_Obj_NameClashing(include_attr);
372
373 // Adjust Dimension name, CHECK: the das generation has a f->HaveUnlimitedDim() condition
374 f->Adjust_Dim_Name();
375 if(General_Product == product_type ||
376 true == HDF5RequestHandler::get_check_name_clashing())
378
379 // Handle the "coordinate" attributes.
380 f->Handle_Coor_Attr();
381
382 f->Handle_Hybrid_EOS5();
383 if(true == f->Have_Grid_Mapping_Attrs())
385#if 0
386 // Need to handle the "coordinate" attributes when memory cache is turned on.
387 if((HDF5RequestHandler::get_lrdata_mem_cache() != nullptr) ||
388 (HDF5RequestHandler::get_srdata_mem_cache() != nullptr))
389 f->Handle_Coor_Attr();
390#endif
391
394
395 if(true == HDF5RequestHandler::get_enable_coord_attr_add_path())
397
398 }
399 catch (HDF5CF::Exception &e){
400 if (f != nullptr)
401 delete f;
402 throw InternalErr(e.what());
403 }
404
405 // generate DMR.
406 try {
407 gen_gmh5_cfdmr(d4_root,f);
408 }
409 catch(...) {
410 if (f != nullptr)
411 delete f;
412 throw;
413 }
414
415 if (f != nullptr)
416 delete f;
417
418}
419
420// Generate DDS mapped from general HDF5 products
421void gen_gmh5_cfdds( DDS & dds, HDF5CF:: GMFile *f) {
422
423 BESDEBUG("h5","Coming to GM DDS generation function gen_gmh5_cfdds() "<<endl);
424
425 const vector<HDF5CF::Var *>& vars = f->getVars();
426 const vector<HDF5CF::GMCVar *>& cvars = f->getCVars();
427 const vector<HDF5CF::GMSPVar *>& spvars = f->getSPVars();
428 const string filename = f->getPath();
429 const hid_t fileid = f->getFileID();
430
431 // Read Variable info.
432
433#if 0
434 //vector<HDF5CF::Var *>::const_iterator it_v;
435 //vector<HDF5CF::GMCVar *>::const_iterator it_cv;
436 //vector<HDF5CF::GMSPVar *>::const_iterator it_spv;
437#endif
438
439 // Since we need to use dds to add das for 64-bit dmr,we need to check if
440 // this case includes 64-bit integer variables and this is for dmr response.
441 bool dmr_64bit_support = false;
442 if(HDF5RequestHandler::get_dmr_long_int()==true &&
443 HDF5RequestHandler::get_dmr_64bit_int()!=nullptr) {
444 for (auto it_v = vars.begin(); it_v !=vars.end();++it_v) {
445 if (H5INT64 == (*it_v)->getType() || H5UINT64 == (*it_v)->getType()){
446 dmr_64bit_support = true;
447 break;
448 }
449 }
450 }
451
452 // We need to remove the unsupported attributes.
453 if(true == dmr_64bit_support) {
454 //STOP: add non-support stuff
455 f->Handle_Unsupported_Dtype(true);
456
457 // Remove unsupported dataspace
458 f->Handle_Unsupported_Dspace(true);
459
460 }
461
462 for (auto it_v = vars.begin(); it_v !=vars.end();++it_v) {
463 BESDEBUG("h5","variable full path= "<< (*it_v)->getFullPath() <<endl);
464 // Handle 64-integer DAP4 CF mapping
465 if(need_attr_values_for_dap4(*it_v) == true)
466 f->Retrieve_H5_Var_Attr_Values(*it_v);
467 gen_dap_onevar_dds(dds,*it_v,fileid, filename);
468 }
469 for (auto it_cv = cvars.begin(); it_cv !=cvars.end();++it_cv) {
470 BESDEBUG("h5","variable full path= "<< (*it_cv)->getFullPath() <<endl);
471 gen_dap_onegmcvar_dds(dds,*it_cv,fileid, filename);
472 }
473
474 for (auto it_spv = spvars.begin(); it_spv !=spvars.end();it_spv++) {
475 BESDEBUG("h5","variable full path= "<< (*it_spv)->getFullPath() <<endl);
476 gen_dap_onegmspvar_dds(dds,*it_spv,fileid, filename);
477 }
478
479}
480
481// Generate DAS mapped from general HDF5 products
482void gen_gmh5_cfdas( DAS & das, HDF5CF:: GMFile *f) {
483
484 BESDEBUG("h5","Coming to GM DAS generation function gen_gmh5_cfdas() "<<endl);
485
486 // First check if this is for generating the ignored object info.
487 if(true == f->Get_IgnoredInfo_Flag()) {
488 gen_gmh5_cf_ignored_obj_info(das, f);
489 return;
490 }
491
492 const vector<HDF5CF::Var *>& vars = f->getVars();
493 const vector<HDF5CF::GMCVar *>& cvars = f->getCVars();
494 const vector<HDF5CF::GMSPVar *>& spvars = f->getSPVars();
495 const vector<HDF5CF::Group *>& grps = f->getGroups();
496 const vector<HDF5CF::Attribute *>& root_attrs = f->getAttributes();
497
498#if 0
499 vector<HDF5CF::Var *>::const_iterator it_v;
500 vector<HDF5CF::GMCVar *>::const_iterator it_cv;
501 vector<HDF5CF::GMSPVar *>::const_iterator it_spv;
502 vector<HDF5CF::Group *>::const_iterator it_g;
503 vector<HDF5CF::Attribute *>::const_iterator it_ra;
504#endif
505
506 // Handling the file attributes(attributes under the root group)
507 // The table name is "HDF_GLOBAL".
508
509 if (false == root_attrs.empty()) {
510
511 AttrTable *at = das.get_table(FILE_ATTR_TABLE_NAME);
512 if (nullptr == at)
513 at = das.add_table(FILE_ATTR_TABLE_NAME, new AttrTable);
514
515 for (auto it_ra = root_attrs.begin(); it_ra != root_attrs.end(); ++it_ra) {
516 // Check and may update the 64-bit integer attributes in DAP4.
517 check_update_int64_attr("",*it_ra);
518 gen_dap_oneobj_das(at,*it_ra,nullptr);
519 }
520 }
521
522 if (false == grps.empty()) {
523 for (auto it_g = grps.begin();
524 it_g != grps.end(); ++it_g) {
525 AttrTable *at = das.get_table((*it_g)->getNewName());
526 if (nullptr == at)
527 at = das.add_table((*it_g)->getNewName(), new AttrTable);
528
529 for (auto it_ra = (*it_g)->getAttributes().begin();
530 it_ra != (*it_g)->getAttributes().end(); ++it_ra) {
531 check_update_int64_attr((*it_g)->getNewName(),*it_ra);
532 gen_dap_oneobj_das(at,*it_ra,nullptr);
533 }
534 }
535 }
536
537 for (auto it_v = vars.begin();
538 it_v != vars.end(); ++it_v) {
539 if (false == ((*it_v)->getAttributes().empty())) {
540
541 // Skip the 64-bit integer variables. The attribute mapping of
542 // DAP4 CF 64-bit integer variable support
543 // has been taken care at the routine gen_dap_onevar_dds()
544 // defined at h5commoncfdap.cc
545 if(H5INT64 == (*it_v)->getType() || H5UINT64 == (*it_v)->getType()){
546 continue;
547 }
548
549 AttrTable *at = das.get_table((*it_v)->getNewName());
550 if (nullptr == at)
551 at = das.add_table((*it_v)->getNewName(), new AttrTable);
552
553 for (auto it_ra = (*it_v)->getAttributes().begin();
554 it_ra != (*it_v)->getAttributes().end(); ++it_ra) {
555 gen_dap_oneobj_das(at,*it_ra,*it_v);
556 }
557 // TODO: If a var has integer-64 bit datatype attributes, maybe
558 // we can just keep that attributes(not consistent but
559 // easy to implement) or we have to duplicate all
560 // the var in dmr and delete this var from dds.
561
562 }
563
564 // GPM needs to be handled in a special way(mostly _FillValue)
565 if(GPMS_L3 == f->getProductType() || GPMM_L3 == f->getProductType()
566 || GPM_L1 == f->getProductType())
567 update_GPM_special_attrs(das,*it_v,false);
568
569 }
570
571 for (auto it_cv = cvars.begin();
572 it_cv != cvars.end(); ++it_cv) {
573 if (false == ((*it_cv)->getAttributes().empty())) {
574
575 // TODO: Add 64-bit int support for coordinates, this has not been tackled.
576 if(H5INT64 == (*it_cv)->getType() || H5UINT64 == (*it_cv)->getType()){
577 continue;
578 }
579
580 AttrTable *at = das.get_table((*it_cv)->getNewName());
581 if (nullptr == at)
582 at = das.add_table((*it_cv)->getNewName(), new AttrTable);
583
584 for (auto it_ra = (*it_cv)->getAttributes().begin();
585 it_ra != (*it_cv)->getAttributes().end(); ++it_ra){
586 gen_dap_oneobj_das(at,*it_ra,*it_cv);
587 }
588
589 }
590 // Though CF doesn't allow _FillValue, still keep it to keep the original form.
591 if(GPMS_L3 == f->getProductType() || GPMM_L3 == f->getProductType()
592 || GPM_L1 == f->getProductType())
593 update_GPM_special_attrs(das,*it_cv,true);
594
595 }
596
597 // Currently the special variables are only limited to the ACOS/OCO2 64-bit integer variables
598 for (auto it_spv = spvars.begin();
599 it_spv != spvars.end(); ++it_spv) {
600 if (false == ((*it_spv)->getAttributes().empty())) {
601
602 AttrTable *at = das.get_table((*it_spv)->getNewName());
603 if (nullptr == at)
604 at = das.add_table((*it_spv)->getNewName(), new AttrTable);
605#if 0
606 // cerr<<"spv coordinate variable name "<<(*it_spv)->getNewName() <<endl;
607#endif
608
609 for (auto it_ra = (*it_spv)->getAttributes().begin();
610 it_ra != (*it_spv)->getAttributes().end(); ++it_ra)
611 gen_dap_oneobj_das(at,*it_ra,*it_spv);
612 }
613 }
614
615 // CHECK ALL UNLIMITED DIMENSIONS from the coordinate variables based on the names.
616 if(f->HaveUnlimitedDim() == true) {
617
618 BESDEBUG("h5","Find unlimited dimension in the GM DAS generation function gen_gmh5_cfdas() "<<endl);
619
620 // Currently there is no way for DAP to present the unlimited dimension info.
621 // when there are no dimension names. So don't create DODS_EXTRA even if
622 // there is an unlimited dimension in the file. KY 2016-02-18
623 if(cvars.empty()==false ){
624
625 // First check if we do have unlimited dimension in the coordinate variables.
626 // Since unsupported fakedims are removed, we may not have unlimited dimensions.
627 bool still_has_unlimited = false;
628 for (auto it_cv = cvars.begin();
629 it_cv != cvars.end(); ++it_cv) {
630 // Check unlimited dimension names.
631 for (auto ird = (*it_cv)->getDimensions().begin();
632 ird != (*it_cv)->getDimensions().end(); ++ird) {
633
634 // Currently we only check one unlimited dimension, which is the most
635 // common case. When receiving the conventions from JG, will add
636 // the support of multi-unlimited dimension. KY 2016-02-09
637 if((*ird)->HaveUnlimitedDim() == true) {
638 still_has_unlimited = true;
639 break;
640 }// if((*ird) is HaveUnlimitedDim()
641 }// for (vector<Dimension*>::
642 if(true == still_has_unlimited)
643 break;
644 }// for (it_cv=cvars.begin();
645
646 if(true == still_has_unlimited) {
647 AttrTable* at = das.get_table("DODS_EXTRA");
648 if (nullptr == at)
649 at = das.add_table("DODS_EXTRA", new AttrTable);
650
651 string unlimited_names;
652
653 for (auto it_cv = cvars.begin();
654 it_cv != cvars.end(); ++it_cv) {
655#if 0
656 bool has_unlimited_dim = false;
657#endif
658 // Check unlimited dimension names.
659 for (auto ird = (*it_cv)->getDimensions().begin();
660 ird != (*it_cv)->getDimensions().end(); ++ird) {
661
662 // Currently we only check one unlimited dimension, which is the most
663 // common case. When receiving the conventions from JG, will add
664 // the support of multi-unlimited dimension. KY 2016-02-09
665 if((*ird)->HaveUnlimitedDim() == true) {
666 if(unlimited_names=="") {
667 unlimited_names = (*ird)->getNewName();
668 if(at !=nullptr)
669 at->append_attr("Unlimited_Dimension","String",unlimited_names);
670 }
671 else {
672 if(unlimited_names.rfind((*ird)->getNewName()) == string::npos) {
673 unlimited_names = unlimited_names+" "+(*ird)->getNewName();
674 if(at !=nullptr)
675 at->append_attr("Unlimited_Dimension","String",(*ird)->getNewName());
676 }
677 }
678 }// if((*ird)->HaveUnlimitedDim()
679 }// for (vector<Dimension*>::
680 }// for (it_cv=cvars.begin();
681 }// if(true == still_has_unlimited)
682
683 }//if(cvars.size()>0)
684#if 0
685 // The following line will generate the string like "Band1 str1 str2".
686 //if(unlimited_names!="")
687 // // at->append_attr("Unlimited_Dimension","String",unlimited_names);
688#endif
689 }
690}
691
692void gen_gmh5_cfdmr(D4Group* d4_root,const HDF5CF::GMFile *f) {
693
694 BESDEBUG("h5","Coming to GM DDS generation function gen_gmh5_cfdmr() "<<endl);
695
696 const vector<HDF5CF::Var *>& vars = f->getVars();
697 const vector<HDF5CF::GMCVar *>& cvars = f->getCVars();
698 const vector<HDF5CF::GMSPVar *>& spvars = f->getSPVars();
699 const string filename = f->getPath();
700 const hid_t fileid = f->getFileID();
701 const vector<HDF5CF::Group *>& grps = f->getGroups();
702 const vector<HDF5CF::Attribute *>& root_attrs = f->getAttributes();
703
704 vector<HDF5CF::Var *>::const_iterator it_v;
705 vector<HDF5CF::GMCVar *>::const_iterator it_cv;
706 vector<HDF5CF::Attribute *>::const_iterator it_ra;
707
708 // Root and low-level group attributes.
709 if (false == root_attrs.empty()) {
710 for (it_ra = root_attrs.begin(); it_ra != root_attrs.end(); ++it_ra)
711 map_cfh5_grp_attr_to_dap4(d4_root,*it_ra);
712 }
713
714 // TODO: Need to wait for HYRAX-687's decision
715 if (HDF5RequestHandler::get_add_dap4_coverage() == true) {
716
717 for (it_cv = cvars.begin(); it_cv !=cvars.end();++it_cv) {
718 BESDEBUG("h5","variable full path= "<< (*it_cv)->getFullPath() <<endl);
719 gen_dap_onegmcvar_dmr(d4_root,*it_cv,fileid, filename);
720 }
721
722 for (it_v = vars.begin(); it_v !=vars.end();++it_v) {
723 BESDEBUG("h5","variable full path= "<< (*it_v)->getFullPath() <<endl);
724 gen_dap_onevar_dmr(d4_root,*it_v,fileid, filename);
725
726 }
727
728
729 }
730 else {
731 // Read Variable info.
732 for (it_v = vars.begin(); it_v !=vars.end();++it_v) {
733 BESDEBUG("h5","variable full path= "<< (*it_v)->getFullPath() <<endl);
734 gen_dap_onevar_dmr(d4_root,*it_v,fileid, filename);
735
736 }
737
738 for (it_cv = cvars.begin(); it_cv !=cvars.end();++it_cv) {
739 BESDEBUG("h5","variable full path= "<< (*it_cv)->getFullPath() <<endl);
740 gen_dap_onegmcvar_dmr(d4_root,*it_cv,fileid, filename);
741 }
742
743 }
744 // GPM needs to be handled in a special way(mostly _FillValue)
745 if(GPMS_L3 == f->getProductType() || GPMM_L3 == f->getProductType()
746 || GPM_L1 == f->getProductType())
747 update_GPM_special_attrs_cfdmr(d4_root,cvars);
748
749
750 for (auto it_spv = spvars.begin(); it_spv !=spvars.end();it_spv++) {
751 BESDEBUG("h5","variable full path= "<< (*it_spv)->getFullPath() <<endl);
752 gen_dap_onegmspvar_dmr(d4_root,*it_spv,fileid, filename);
753 }
754
755 // We use the attribute container to store the group attributes.
756 if (false == grps.empty()) {
757 for (auto it_g = grps.begin();
758 it_g != grps.end(); ++it_g) {
759
760 auto tmp_grp = new D4Attribute;
761 tmp_grp->set_name((*it_g)->getNewName());
762
763 // Make the type as a container
764 tmp_grp->set_type(attr_container_c);
765
766 for (it_ra = (*it_g)->getAttributes().begin();
767 it_ra != (*it_g)->getAttributes().end(); ++it_ra) {
768 map_cfh5_attr_container_to_dap4(tmp_grp,(*it_ra));
769 }
770
771 d4_root->attributes()->add_attribute_nocopy(tmp_grp);
772 }
773 }
774
775 // CHECK ALL UNLIMITED DIMENSIONS from the coordinate variables based on the names.
776 if(f->HaveUnlimitedDim() == true) {
777
778 BESDEBUG("h5","Find unlimited dimension in the GM DMR generation function gen_gmh5_cfdmr() "<<endl);
779
780 // Currently there is no way for DAP to present the unlimited dimension info.
781 // when there are no dimension names. So don't create DODS_EXTRA even if
782 // there is an unlimited dimension in the file. KY 2016-02-18
783 if(cvars.empty()==false ){
784
785 // First check if we do have unlimited dimension in the coordinate variables.
786 // Since unsupported fakedims are removed, we may not have unlimited dimensions.
787 bool still_has_unlimited = false;
788 for (it_cv = cvars.begin();
789 it_cv != cvars.end(); ++it_cv) {
790 // Check unlimited dimension names.
791 for (auto ird = (*it_cv)->getDimensions().begin();
792 ird != (*it_cv)->getDimensions().end(); ++ird) {
793
794 // Currently we only check one unlimited dimension, which is the most
795 // common case. When receiving the conventions from JG, will add
796 // the support of multi-unlimited dimension. KY 2016-02-09
797 if((*ird)->HaveUnlimitedDim() == true) {
798 still_has_unlimited = true;
799 break;
800 }// if((*ird) is HaveUnlimitedDim()
801 }// for (vector<Dimension*>::
802 if(true == still_has_unlimited)
803 break;
804 }// for (it_cv=cvars.begin();
805
806 if(true == still_has_unlimited) {
807
808 string dods_extra = "DODS_EXTRA";
809
810 // If DODS_EXTRA exists, we will not create the unlimited dimensions.
811 if(d4_root->attributes() != nullptr) {
812 // The following lines cause seg. fault in libdap4, needs to investigate
813 //if((d4_root->attributes()->find(dods_extra))==nullptr)
814
815 string unlimited_dim_names ="";
816
817 for (it_cv = cvars.begin();
818 it_cv != cvars.end(); it_cv++) {
819
820 // Check unlimited dimension names.
821 for (auto ird = (*it_cv)->getDimensions().begin();
822 ird != (*it_cv)->getDimensions().end(); ++ird) {
823
824 // Currently we only check one unlimited dimension, which is the most
825 // common case. When receiving the conventions from JG, will add
826 // the support of multi-unlimited dimension. KY 2016-02-09
827 if((*ird)->HaveUnlimitedDim() == true) {
828
829 string unlimited_dim_name = (*ird)->getNewName();
830 if(unlimited_dim_names=="")
831 unlimited_dim_names = unlimited_dim_name;
832 else {
833 if(unlimited_dim_names.rfind(unlimited_dim_name) == string::npos)
834 unlimited_dim_names = unlimited_dim_names+" "+unlimited_dim_name;
835 }
836 }
837 }
838 }
839
840 if(unlimited_dim_names != "") {
841 auto dods_extra_attr = new D4Attribute(dods_extra,attr_container_c);
842 auto unlimited_dim_attr = new D4Attribute("Unlimited_Dimension",attr_str_c);
843 unlimited_dim_attr->add_value(unlimited_dim_names);
844 dods_extra_attr->attributes()->add_attribute_nocopy(unlimited_dim_attr);
845 d4_root->attributes()->add_attribute_nocopy(dods_extra_attr);
846
847 }
848 else
849 throw InternalErr(__FILE__, __LINE__, "Unlimited dimension should exist.");
850 }
851
852 }
853 }
854 }
855
856
857 // Add DAP4 Map for coverage
858 if (HDF5RequestHandler::get_add_dap4_coverage() == true) {
859
860 // Obtain the coordinate variable names, these are mapped variables.
861 vector <string> cvar_name;
862 for (it_cv = cvars.begin(); it_cv !=cvars.end();++it_cv)
863 cvar_name.emplace_back((*it_cv)->getNewName());
864
865 add_dap4_coverage(d4_root,cvar_name,f->getIsCOARD());
866 }
867
868}
869
870// Generate the ignored object info. for the CF option of the general products
871void gen_gmh5_cf_ignored_obj_info(DAS &das, HDF5CF::GMFile *f) {
872
873 BESDEBUG("h5","Coming to gen_gmh5_cf_ignored_obj_info() "<<endl);
874 AttrTable *at = das.get_table("Ignored_Object_Info");
875 if (nullptr == at)
876 at = das.add_table("Ignored_Object_Info", new AttrTable);
877
878 at->append_attr("Message","String",f->Get_Ignored_Msg());
879
880}
881
882// Generate the DDS for a coordinate variable of the General products
883void gen_dap_onegmcvar_dds(DDS &dds,const HDF5CF::GMCVar* cvar, const hid_t file_id, const string & filename) {
884
885 BESDEBUG("h5","Coming to gen_dap_onegmcvar_dds() "<<endl);
886
887 if(cvar->getType() == H5INT64 || cvar->getType() == H5UINT64)
888 return;
889 BaseType *bt = nullptr;
890
891 switch(cvar->getType()) {
892#define HANDLE_CASE(tid,type) \
893 case tid: \
894 bt = new (type)(cvar->getNewName(),cvar->getFullPath()); \
895 break;
896
897 HANDLE_CASE(H5FLOAT32, HDF5CFFloat32)
898 HANDLE_CASE(H5FLOAT64, HDF5CFFloat64)
899 HANDLE_CASE(H5CHAR,HDF5CFInt16)
900 HANDLE_CASE(H5UCHAR, HDF5CFByte)
901 HANDLE_CASE(H5INT16, HDF5CFInt16)
902 HANDLE_CASE(H5UINT16, HDF5CFUInt16)
903 HANDLE_CASE(H5INT32, HDF5CFInt32)
904 HANDLE_CASE(H5UINT32, HDF5CFUInt32)
905 HANDLE_CASE(H5FSTRING, Str)
906 HANDLE_CASE(H5VSTRING, Str)
907
908 default:
909 throw InternalErr(__FILE__,__LINE__,"unsupported data type.");
910#undef HANDLE_CASE
911 }
912
913 if (bt) {
914
915 const vector<HDF5CF::Dimension *>& dims = cvar->getDimensions();
916 vector <HDF5CF::Dimension*>:: const_iterator it_d;
917 vector <size_t> dimsizes;
918 dimsizes.resize(cvar->getRank());
919 for(int i = 0; i <cvar->getRank();i++)
920 dimsizes[i] = (dims[i])->getSize();
921
922
923 if(dims.empty())
924 throw InternalErr(__FILE__,__LINE__,"the coordinate variable cannot be a scalar");
925
926 switch(cvar->getCVType()) {
927
928 case CV_EXIST:
929 {
930 HDF5CFArray *ar = nullptr;
931
932 // Need to check if this CV is lat/lon. This is necessary when data memory cache is turned on.
933 bool is_latlon = cvar->isLatLon();
934
935 try {
936 ar = new HDF5CFArray (
937 cvar->getRank(),
938 file_id,
939 filename,
940 cvar->getType(),
941 dimsizes,
942 cvar->getFullPath(),
943 cvar->getTotalElems(),
944 CV_EXIST,
945 is_latlon,
946 cvar->getCompRatio(),
947 false,
948 cvar->getNewName(),
949 bt);
950 }
951 catch(...) {
952 delete bt;
953 throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5CFArray. ");
954 }
955
956 for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
957 if (""==(*it_d)->getNewName())
958 ar->append_dim((int)((*it_d)->getSize()));
959 else
960 ar->append_dim((int)((*it_d)->getSize()), (*it_d)->getNewName());
961 }
962
963 dds.add_var(ar);
964 delete bt;
965 delete ar;
966 }
967 break;
968
969 case CV_LAT_MISS:
970 case CV_LON_MISS:
971 {
972 // Using HDF5GMCFMissLLArray
973 HDF5GMCFMissLLArray *ar = nullptr;
974 try {
975 ar = new HDF5GMCFMissLLArray (
976 cvar->getRank(),
977 filename,
978 file_id,
979 cvar->getType(),
980 cvar->getFullPath(),
981 cvar->getPtType(),
982 cvar->getCVType(),
983 cvar->getNewName(),
984 bt);
985 }
986 catch(...) {
987 delete bt;
988 throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissLLArray. ");
989 }
990
991
992 for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
993 if (""==(*it_d)->getNewName())
994 ar->append_dim((int)((*it_d)->getSize()));
995 else
996 ar->append_dim((int)((*it_d)->getSize()), (*it_d)->getNewName());
997 }
998
999 dds.add_var(ar);
1000 delete bt;
1001 delete ar;
1002 }
1003 break;
1004
1005 case CV_NONLATLON_MISS:
1006 {
1007
1008 if (cvar->getRank() !=1) {
1009 delete bt;
1010 throw InternalErr(__FILE__, __LINE__, "The rank of missing Z dimension field must be 1");
1011 }
1012 auto nelem = (int)((cvar->getDimensions()[0])->getSize());
1013
1014 HDF5GMCFMissNonLLCVArray *ar = nullptr;
1015
1016 try {
1017 ar = new HDF5GMCFMissNonLLCVArray(
1018 cvar->getRank(),
1019 nelem,
1020 cvar->getNewName(),
1021 bt);
1022 }
1023 catch(...) {
1024 delete bt;
1025 throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissNonLLCVArray. ");
1026 }
1027
1028
1029 for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
1030 if (""==(*it_d)->getNewName())
1031 ar->append_dim((int)((*it_d)->getSize()));
1032 else
1033 ar->append_dim((int)((*it_d)->getSize()), (*it_d)->getNewName());
1034 }
1035 dds.add_var(ar);
1036 delete bt;
1037 delete ar;
1038 }
1039 break;
1040
1041 case CV_FILLINDEX:
1042 {
1043
1044 if (cvar->getRank() !=1) {
1045 delete bt;
1046 throw InternalErr(__FILE__, __LINE__, "The rank of missing Z dimension field must be 1");
1047 }
1048
1049 HDF5GMCFFillIndexArray *ar = nullptr;
1050
1051 try {
1052 ar = new HDF5GMCFFillIndexArray(
1053 cvar->getRank(),
1054 cvar->getType(),
1055 false,
1056 cvar->getNewName(),
1057 bt);
1058 }
1059 catch(...) {
1060 delete bt;
1061 throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissNonLLCVArray. ");
1062 }
1063
1064
1065 for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
1066 if (""==(*it_d)->getNewName())
1067 ar->append_dim((int)((*it_d)->getSize()));
1068 else
1069 ar->append_dim((int)((*it_d)->getSize()), (*it_d)->getNewName());
1070 }
1071 dds.add_var(ar);
1072 delete bt;
1073 delete ar;
1074 }
1075 break;
1076
1077
1078 case CV_SPECIAL:
1079 {
1080 // Currently only handle 1-D special CV.
1081 if (cvar->getRank() !=1) {
1082 delete bt;
1083 throw InternalErr(__FILE__, __LINE__, "The rank of special coordinate variable must be 1");
1084 }
1085 auto nelem = (int)((cvar->getDimensions()[0])->getSize());
1086
1087 HDF5GMCFSpecialCVArray * ar = nullptr;
1088 ar = new HDF5GMCFSpecialCVArray(
1089 cvar->getType(),
1090 nelem,
1091 cvar->getFullPath(),
1092 cvar->getPtType(),
1093 cvar->getNewName(),
1094 bt);
1095
1096 for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
1097 if (""==(*it_d)->getNewName())
1098 ar->append_dim((int)((*it_d)->getSize()));
1099 else
1100 ar->append_dim((int)((*it_d)->getSize()), (*it_d)->getNewName());
1101 }
1102
1103 dds.add_var(ar);
1104 delete bt;
1105 delete ar;
1106
1107 }
1108 break;
1109 case CV_MODIFY:
1110 default:
1111 delete bt;
1112 throw InternalErr(__FILE__,__LINE__,"Coordinate variable type is not supported.");
1113 }
1114 }
1115}
1116
1117// Generate DDS for special variable in a general product
1118void gen_dap_onegmspvar_dds(DDS &dds,const HDF5CF::GMSPVar* spvar, const hid_t fileid, const string & filename) {
1119
1120 BESDEBUG("h5","Coming to gen_dap_onegmspvar_dds() "<<endl);
1121 BaseType *bt = nullptr;
1122
1123 switch(spvar->getType()) {
1124#define HANDLE_CASE(tid,type) \
1125 case tid: \
1126 bt = new (type)(spvar->getNewName(),spvar->getFullPath()); \
1127 break;
1128
1129 HANDLE_CASE(H5FLOAT32, HDF5CFFloat32)
1130 HANDLE_CASE(H5FLOAT64, HDF5CFFloat64)
1131 HANDLE_CASE(H5CHAR,HDF5CFInt16)
1132 HANDLE_CASE(H5UCHAR, HDF5CFByte)
1133 HANDLE_CASE(H5INT16, HDF5CFInt16)
1134 HANDLE_CASE(H5UINT16, HDF5CFUInt16)
1135 HANDLE_CASE(H5INT32, HDF5CFInt32)
1136 HANDLE_CASE(H5UINT32, HDF5CFUInt32)
1137 HANDLE_CASE(H5FSTRING, Str)
1138 HANDLE_CASE(H5VSTRING, Str)
1139 default:
1140 throw InternalErr(__FILE__,__LINE__,"unsupported data type.");
1141#undef HANDLE_CASE
1142 }
1143
1144 if (bt) {
1145
1146 const vector<HDF5CF::Dimension *>& dims = spvar->getDimensions();
1147
1148 if(dims.empty())
1149 throw InternalErr(__FILE__,__LINE__,"Currently don't support scalar special variables. ");
1150
1151 HDF5GMSPCFArray *ar = nullptr;
1152
1153 try {
1154 ar = new HDF5GMSPCFArray (
1155 spvar->getRank(),
1156 filename,
1157 fileid,
1158 spvar->getType(),
1159 spvar->getFullPath(),
1160 spvar->getOriginalType(),
1161 spvar->getStartBit(),
1162 spvar->getBitNum(),
1163 spvar->getNewName(),
1164 bt);
1165 }
1166 catch(...) {
1167 delete bt;
1168 throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissNonLLCVArray. ");
1169 }
1170
1171
1172 for(auto it_d = dims.begin(); it_d != dims.end(); ++it_d) {
1173 if (""==(*it_d)->getNewName())
1174 ar->append_dim((int)((*it_d)->getSize()));
1175 else
1176 ar->append_dim((int)((*it_d)->getSize()), (*it_d)->getNewName());
1177 }
1178
1179 dds.add_var(ar);
1180 delete bt;
1181 delete ar;
1182 }
1183
1184}
1185
1186// When we add floating point fill value at HDF5CF.cc, the value will be changed
1187// a little bit when it changes to string representation.
1188// For example, -9999.9 becomes -9999.9000123. To reduce the misunderstanding,we
1189// just add fillvalue in the string type here. KY 2014-04-02
1190void update_GPM_special_attrs(DAS& das, const HDF5CF::Var *var,bool is_cvar) {
1191
1192 BESDEBUG("h5","Coming to update_GPM_special_attrs() "<<endl);
1193 if(H5FLOAT64 == var->getType() ||
1194 H5FLOAT32 == var->getType() ||
1195 H5INT16 == var->getType() ||
1196 H5CHAR == var->getType()) {
1197
1198 AttrTable *at = das.get_table(var->getNewName());
1199 if (nullptr == at)
1200 at = das.add_table(var->getNewName(), new AttrTable);
1201 bool has_fillvalue = false;
1202 AttrTable::Attr_iter it = at->attr_begin();
1203 while (it!=at->attr_end() && false==has_fillvalue) {
1204 if (at->get_name(it) =="_FillValue")
1205 {
1206 has_fillvalue = true;
1207 string fillvalue ="";
1208 if(H5FLOAT32 == var->getType()) {
1209 const string cor_fill_value = "-9999.9";
1210 fillvalue = (*at->get_attr_vector(it)->begin());
1211 if((fillvalue.find(cor_fill_value) == 0) && (fillvalue!= cor_fill_value)) {
1212 at->del_attr("_FillValue");
1213 at->append_attr("_FillValue","Float32",cor_fill_value);
1214 }
1215 }
1216 else if(H5FLOAT64 == var->getType()) {
1217 const string cor_fill_value = "-9999.9";
1218 const string exist_fill_value_substr = "-9999.8999";
1219 fillvalue = (*at->get_attr_vector(it)->begin());
1220 if((fillvalue.find(exist_fill_value_substr) == 0) && (fillvalue!= cor_fill_value)) {
1221 at->del_attr("_FillValue");
1222 at->append_attr("_FillValue","Float64",cor_fill_value);
1223 }
1224
1225 }
1226 }
1227 it++;
1228 }
1229
1230 // Add the fill value
1231 if(false == is_cvar ) {
1232
1233 // Current versions of GPM don't add fillvalues. We add the fillvalue according to the document.
1234 if (has_fillvalue != true ) {
1235
1236 if(H5FLOAT32 == var->getType())
1237 at->append_attr("_FillValue","Float32","-9999.9");
1238 else if(H5FLOAT64 == var->getType())
1239 at->append_attr("_FillValue","Float64","-9999.9");
1240 else if (H5INT16 == var->getType())
1241 at->append_attr("_FillValue","Int16","-9999");
1242 else if (H5CHAR == var->getType())// H5CHAR maps to DAP int16
1243 at->append_attr("_FillValue","Int16","-99");
1244
1245 }
1246 }
1247 }
1248}
1249
1250// This routine is following the DAP2 way,except we map HDF5 8-bit integer to DAP4 8-bit integer.
1251// When we add floating point fill value at HDF5CF.cc, the value will be changed
1252// a little bit when it changes to string representation.
1253// For example, -9999.9 becomes -9999.9000123. To reduce the misunderstanding,we
1254// just add fillvalue in the string type here. KY 2014-04-02
1255void update_GPM_special_attrs_cfdmr(libdap::D4Group* d4_root, const vector<HDF5CF::GMCVar *>& cvars) {
1256
1257 BESDEBUG("h5","Coming to update_GPM_special_attrs_cfdmr() "<<endl);
1258
1259 // We need to loop through all the variables.
1260 Constructor::Vars_iter vi = d4_root->var_begin();
1261 Constructor::Vars_iter ve = d4_root->var_end();
1262
1263 for (; vi != ve; vi++) {
1264
1265 // The _FillValue datatype only applies to
1266 // 32-bit and 64-bit floating-point data and
1267 // 8-bit and 16-bit integer.
1268
1269 Type var_type = (*vi)->type();
1270 if ((*vi)->type() == dods_array_c)
1271 var_type = (*vi)->var()->type();
1272 if (dods_float64_c == var_type ||
1273 dods_float32_c == var_type ||
1274 dods_int16_c == var_type ||
1275 dods_int8_c == var_type) {
1276
1277 D4Attribute *d4_attr = (*vi)->attributes()->find("_FillValue");
1278
1279 // If we don't find the _FillValue, according to DAP2 implementation,
1280 // we need to add the corresponding fill values.
1281 if(!d4_attr) {
1282 bool is_cvar = false;
1283 for (auto it_cv = cvars.begin(); it_cv !=cvars.end();++it_cv) {
1284 if ((*it_cv)->getNewName() == (*vi)->name()) {
1285 is_cvar = true;
1286 break;
1287 }
1288 }
1289
1290 // Add fillvalue for real variables not for the coordinate variables.
1291 if(false == is_cvar) {
1292 // Add a DAP4 attribute
1293 D4Attribute *d4_fv = nullptr;
1294 if (dods_float64_c == var_type ) {
1295 d4_fv = new D4Attribute("_FillValue",attr_float64_c);
1296 d4_fv->add_value("-9999.9");
1297 }
1298 else if (dods_float32_c == var_type) {
1299 d4_fv = new D4Attribute("_FillValue",attr_float32_c);
1300 d4_fv->add_value("-9999.9");
1301 }
1302 else if (dods_int16_c == var_type) {
1303 d4_fv = new D4Attribute("_FillValue",attr_int16_c);
1304 d4_fv->add_value("-9999");
1305 }
1306 else if (dods_int8_c == var_type) {
1307 d4_fv = new D4Attribute("_FillValue",attr_int8_c);
1308 d4_fv->add_value("-99");
1309 }
1310 (*vi)->attributes()->add_attribute_nocopy(d4_fv);
1311 }
1312 }
1313//#if 0
1314 else {
1315 D4Attribute *d4_fv = nullptr;
1316 if (dods_float64_c == var_type ) {
1317 const string cor_fill_value = "-9999.9";
1318 const string exist_fill_value_substr = "-9999.8999";
1319 string fillvalue = d4_attr->value(0);
1320 if((fillvalue.find(exist_fill_value_substr) == 0) && (fillvalue!= cor_fill_value)) {
1321 (*vi)->attributes()->erase("_FillValue");
1322 d4_fv = new D4Attribute("_FillValue",attr_float64_c);
1323 d4_fv->add_value(cor_fill_value);
1324 (*vi)->attributes()->add_attribute_nocopy(d4_fv);
1325 }
1326 }
1327 else if (dods_float32_c == var_type) {
1328 const string cor_fill_value = "-9999.9";
1329 string fillvalue = d4_attr->value(0);
1330 // Somehow the fillvalue changes to "-9999.90??", we want to turn it back.
1331 if((fillvalue.find(cor_fill_value) == 0) && (fillvalue!= cor_fill_value)) {
1332 (*vi)->attributes()->erase("_FillValue");
1333 d4_fv = new D4Attribute("_FillValue",attr_float32_c);
1334 d4_fv->add_value(cor_fill_value);
1335 (*vi)->attributes()->add_attribute_nocopy(d4_fv);
1336 }
1337 }
1338 }
1339//#endif
1340 }
1341 }
1342
1343}
1344
1345void gen_dap_onegmcvar_dmr(D4Group*d4_root,const GMCVar* cvar,const hid_t fileid, const string &filename) {
1346
1347 BESDEBUG("h5","Coming to gen_dap_onegmcvar_dds() "<<endl);
1348
1349 BaseType *bt = nullptr;
1350
1351 switch(cvar->getType()) {
1352#define HANDLE_CASE(tid,type) \
1353 case tid: \
1354 bt = new (type)(cvar->getNewName(),cvar->getFullPath()); \
1355 break;
1356
1357 HANDLE_CASE(H5FLOAT32, HDF5CFFloat32)
1358 HANDLE_CASE(H5FLOAT64, HDF5CFFloat64)
1359 HANDLE_CASE(H5CHAR,HDF5CFInt8)
1360 HANDLE_CASE(H5UCHAR, HDF5CFByte)
1361 HANDLE_CASE(H5INT16, HDF5CFInt16)
1362 HANDLE_CASE(H5UINT16, HDF5CFUInt16)
1363 HANDLE_CASE(H5INT32, HDF5CFInt32)
1364 HANDLE_CASE(H5UINT32, HDF5CFUInt32)
1365 HANDLE_CASE(H5INT64, HDF5CFInt64)
1366 HANDLE_CASE(H5UINT64, HDF5CFUInt64)
1367 HANDLE_CASE(H5FSTRING, Str)
1368 HANDLE_CASE(H5VSTRING, Str)
1369
1370 default:
1371 throw InternalErr(__FILE__,__LINE__,"unsupported data type.");
1372#undef HANDLE_CASE
1373 }
1374
1375 if (bt) {
1376
1377 const vector<HDF5CF::Dimension *>& dims = cvar->getDimensions();
1378 vector <HDF5CF::Dimension*>:: const_iterator it_d;
1379 vector <size_t> dimsizes;
1380 dimsizes.resize(cvar->getRank());
1381
1382 for(int i = 0; i <cvar->getRank();i++)
1383 dimsizes[i] = (dims[i])->getSize();
1384
1385 if(dims.empty())
1386 throw InternalErr(__FILE__,__LINE__,"the coordinate variable cannot be a scalar");
1387
1388 switch(cvar->getCVType()) {
1389
1390 case CV_EXIST:
1391 {
1392 HDF5CFArray *ar = nullptr;
1393
1394 // Need to check if this CV is lat/lon. This is necessary when data memory cache is turned on.
1395 bool is_latlon = cvar->isLatLon();
1396
1397 try {
1398
1399 bool is_dap4 = true;
1400 ar = new HDF5CFArray (
1401 cvar->getRank(),
1402 fileid,
1403 filename,
1404 cvar->getType(),
1405 dimsizes,
1406 cvar->getFullPath(),
1407 cvar->getTotalElems(),
1408 CV_EXIST,
1409 is_latlon,
1410 cvar->getCompRatio(),
1411 is_dap4,
1412 cvar->getNewName(),
1413 bt);
1414 }
1415 catch(...) {
1416 delete bt;
1417 throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5CFArray. ");
1418 }
1419
1420 for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
1421 if (""==(*it_d)->getNewName())
1422 ar->append_dim((int)((*it_d)->getSize()));
1423 else
1424 ar->append_dim((int)((*it_d)->getSize()), (*it_d)->getNewName());
1425 }
1426
1427 ar->set_is_dap4(true);
1428 BaseType* d4_var=ar->h5cfdims_transform_to_dap4(d4_root);
1429 map_cfh5_var_attrs_to_dap4(cvar,d4_var);
1430 d4_root->add_var_nocopy(d4_var);
1431 delete bt;
1432 delete ar;
1433 }
1434 break;
1435
1436 case CV_LAT_MISS:
1437 case CV_LON_MISS:
1438 {
1439 // Using HDF5GMCFMissLLArray
1440 HDF5GMCFMissLLArray *ar = nullptr;
1441 try {
1442 ar = new HDF5GMCFMissLLArray (
1443 cvar->getRank(),
1444 filename,
1445 fileid,
1446 cvar->getType(),
1447 cvar->getFullPath(),
1448 cvar->getPtType(),
1449 cvar->getCVType(),
1450 cvar->getNewName(),
1451 bt);
1452 }
1453 catch(...) {
1454 delete bt;
1455 throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissLLArray. ");
1456 }
1457
1458 for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
1459 if (""==(*it_d)->getNewName())
1460 ar->append_dim((int)((*it_d)->getSize()));
1461 else
1462 ar->append_dim((int)((*it_d)->getSize()), (*it_d)->getNewName());
1463 }
1464
1465 ar->set_is_dap4(true);
1466 BaseType* d4_var=ar->h5cfdims_transform_to_dap4(d4_root);
1467 map_cfh5_var_attrs_to_dap4(cvar,d4_var);
1468 d4_root->add_var_nocopy(d4_var);
1469 delete bt;
1470 delete ar;
1471 }
1472 break;
1473
1474 case CV_NONLATLON_MISS:
1475 {
1476
1477 if (cvar->getRank() !=1) {
1478 delete bt;
1479 throw InternalErr(__FILE__, __LINE__, "The rank of missing Z dimension field must be 1");
1480 }
1481 auto nelem = (int)((cvar->getDimensions()[0])->getSize());
1482
1483 HDF5GMCFMissNonLLCVArray *ar = nullptr;
1484
1485 try {
1486 ar = new HDF5GMCFMissNonLLCVArray(
1487 cvar->getRank(),
1488 nelem,
1489 cvar->getNewName(),
1490 bt);
1491 }
1492 catch(...) {
1493 delete bt;
1494 throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissNonLLCVArray. ");
1495 }
1496
1497
1498 for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
1499 if (""==(*it_d)->getNewName())
1500 ar->append_dim((int)((*it_d)->getSize()));
1501 else
1502 ar->append_dim((int)((*it_d)->getSize()), (*it_d)->getNewName());
1503 }
1504 ar->set_is_dap4(true);
1505 BaseType* d4_var=ar->h5cfdims_transform_to_dap4(d4_root);
1506 map_cfh5_var_attrs_to_dap4(cvar,d4_var);
1507 d4_root->add_var_nocopy(d4_var);
1508 delete bt;
1509 delete ar;
1510 }
1511 break;
1512
1513 case CV_FILLINDEX:
1514 {
1515
1516 if (cvar->getRank() !=1) {
1517 delete bt;
1518 throw InternalErr(__FILE__, __LINE__, "The rank of missing Z dimension field must be 1");
1519 }
1520
1521 HDF5GMCFFillIndexArray *ar = nullptr;
1522
1523 try {
1524 ar = new HDF5GMCFFillIndexArray(
1525 cvar->getRank(),
1526 cvar->getType(),
1527 true,
1528 cvar->getNewName(),
1529 bt);
1530 }
1531 catch(...) {
1532 delete bt;
1533 throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissNonLLCVArray. ");
1534 }
1535
1536
1537 for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
1538 if (""==(*it_d)->getNewName())
1539 ar->append_dim((int)((*it_d)->getSize()));
1540 else
1541 ar->append_dim((int)((*it_d)->getSize()), (*it_d)->getNewName());
1542 }
1543 ar->set_is_dap4(true);
1544 BaseType* d4_var=ar->h5cfdims_transform_to_dap4(d4_root);
1545 map_cfh5_var_attrs_to_dap4(cvar,d4_var);
1546 d4_root->add_var_nocopy(d4_var);
1547 delete bt;
1548 delete ar;
1549 }
1550 break;
1551
1552
1553 case CV_SPECIAL:
1554 {
1555 // Currently only handle 1-D special CV.
1556 if (cvar->getRank() !=1) {
1557 delete bt;
1558 throw InternalErr(__FILE__, __LINE__, "The rank of special coordinate variable must be 1");
1559 }
1560 int nelem = (cvar->getDimensions()[0])->getSize();
1561
1562 HDF5GMCFSpecialCVArray * ar = nullptr;
1563 ar = new HDF5GMCFSpecialCVArray(
1564 cvar->getType(),
1565 nelem,
1566 cvar->getFullPath(),
1567 cvar->getPtType(),
1568 cvar->getNewName(),
1569 bt);
1570
1571 for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
1572 if (""==(*it_d)->getNewName())
1573 ar->append_dim((int)((*it_d)->getSize()));
1574 else
1575 ar->append_dim((int)((*it_d)->getSize()), (*it_d)->getNewName());
1576 }
1577
1578 ar->set_is_dap4(true);
1579 BaseType* d4_var=ar->h5cfdims_transform_to_dap4(d4_root);
1580 map_cfh5_var_attrs_to_dap4(cvar,d4_var);
1581 d4_root->add_var_nocopy(d4_var);
1582 delete bt;
1583 delete ar;
1584
1585 }
1586 break;
1587 case CV_MODIFY:
1588 default:
1589 delete bt;
1590 throw InternalErr(__FILE__,__LINE__,"Coordinate variable type is not supported.");
1591 }
1592 }
1593
1594
1595}
1596
1597void gen_dap_onegmspvar_dmr(D4Group*d4_root,const GMSPVar*spvar,const hid_t fileid, const string &filename) {
1598
1599 BESDEBUG("h5","Coming to gen_dap_onegmspvar_dmr() "<<endl);
1600 BaseType *bt = nullptr;
1601
1602 // Note: The special variable is actually an ACOS_OCO2 64-bit integer variable.
1603 // We decompose 64-bit to two integer variables according to the specification.
1604 // This product has been served in this way for years. For backward compatibility,
1605 // we will not change this in the CF DMR implementation. So Int64/UInt64 are not added.
1606 // KY 2021-03-09
1607 switch(spvar->getType()) {
1608#define HANDLE_CASE(tid,type) \
1609 case tid: \
1610 bt = new (type)(spvar->getNewName(),spvar->getFullPath()); \
1611 break;
1612
1613 HANDLE_CASE(H5FLOAT32, HDF5CFFloat32)
1614 HANDLE_CASE(H5FLOAT64, HDF5CFFloat64)
1615 HANDLE_CASE(H5CHAR,HDF5CFByte)
1616 HANDLE_CASE(H5UCHAR, HDF5CFByte)
1617 HANDLE_CASE(H5INT16, HDF5CFInt16)
1618 HANDLE_CASE(H5UINT16, HDF5CFUInt16)
1619 HANDLE_CASE(H5INT32, HDF5CFInt32)
1620 HANDLE_CASE(H5UINT32, HDF5CFUInt32)
1621 HANDLE_CASE(H5FSTRING, Str)
1622 HANDLE_CASE(H5VSTRING, Str)
1623 default:
1624 throw InternalErr(__FILE__,__LINE__,"unsupported data type.");
1625#undef HANDLE_CASE
1626 }
1627
1628 if (bt) {
1629
1630 const vector<HDF5CF::Dimension *>& dims = spvar->getDimensions();
1631
1632 if(dims.empty())
1633 throw InternalErr(__FILE__,__LINE__,"Currently don't support scalar special variables. ");
1634
1635 HDF5GMSPCFArray *ar = nullptr;
1636
1637 try {
1638 ar = new HDF5GMSPCFArray (
1639 spvar->getRank(),
1640 filename,
1641 fileid,
1642 spvar->getType(),
1643 spvar->getFullPath(),
1644 spvar->getOriginalType(),
1645 spvar->getStartBit(),
1646 spvar->getBitNum(),
1647 spvar->getNewName(),
1648 bt);
1649 }
1650 catch(...) {
1651 delete bt;
1652 throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissNonLLCVArray. ");
1653 }
1654
1655
1656 for(auto it_d = dims.begin(); it_d != dims.end(); ++it_d) {
1657 if (""==(*it_d)->getNewName())
1658 ar->append_dim((int)((*it_d)->getSize()));
1659 else
1660 ar->append_dim((int)((*it_d)->getSize()), (*it_d)->getNewName());
1661 }
1662
1663 ar->set_is_dap4(true);
1664 BaseType* d4_var=ar->h5cfdims_transform_to_dap4(d4_root);
1665 map_cfh5_var_attrs_to_dap4(spvar,d4_var);
1666 d4_root->add_var_nocopy(d4_var);
1667
1668 delete bt;
1669 delete ar;
1670 }
1671
1672}
This class includes the methods to read data array into DAP buffer from an HDF5 dataset for the CF op...
This class provides a way to map HDF5 byte to DAP byte for the CF option.
This class provides a way to map HDF5 float to DAP float for the CF option.
This class provides a way to map HDF5 64-bit floating-point(double) to DAP 64-bit floating-point for ...
This class provides a way to map HDF5 int16 to DAP int16 for the CF option.
This class provides a way to map HDF5 32-bit integer to DAP Int32 for the CF option.
This class provides a way to map HDF5 64-bit integer to DAP4 Int64 for the CF option.
This class provides a way to map HDF5 int8 to DAP int16 for the CF option.
This class provides a way to map HDF5 Str to DAP Str for the CF option.
This class provides a way to map HDF5 unsigned 16-bit integer to DAP uint16 for the CF option.
This class provides a way to map HDF5 unsigned 32-bit integer to DAP uint32 for the CF option.
This class provides a way to map HDF5 64-bit unsigned integer to DAP4 UInt64 for the CF option.
This class includes the methods to read data array into DAP buffer from an HDF5 dataset for the CF op...
This class specifies the retrieval of the missing lat/lon values for general HDF5 products.
This class specifies the retrieval of the values of non-lat/lon coordinate variables for general HDF5...
This class specifies the retrieval of the missing lat/lon values for general HDF5 products.
This class specifies the retrieval of data values for special HDF5 products Currently this only appli...
include the entry functions to execute the handlers
CVType getCVType() const
Get the coordinate variable type of this variable.
Definition: HDF5CF.h:362
bool HaveUnlimitedDim() const
Has unlimited dimensions.
Definition: HDF5CF.h:661
hid_t getFileID() const
Obtain the HDF5 file ID.
Definition: HDF5CF.h:631
const std::vector< Attribute * > & getAttributes() const
Public interface to obtain information of all attributes under the root group.
Definition: HDF5CF.h:649
const std::vector< Group * > & getGroups() const
Public interface to obtain all the group info.
Definition: HDF5CF.h:655
const std::string & getPath() const
Obtain the path of the file.
Definition: HDF5CF.h:637
const std::vector< Var * > & getVars() const
Public interface to obtain information of all variables.
Definition: HDF5CF.h:643
This class is a derived class of CVar. It represents a coordinate variable for general HDF5 files.
Definition: HDF5CF.h:413
H5GCFProduct getPtType() const
Get the data type of this variable.
Definition: HDF5CF.h:420
This class is a derived class of File. It includes methods applied to general HDF5 files only.
Definition: HDF5CF.h:826
void Add_Supplement_Attrs(bool) override
Add supplemental attributes such as fullpath and original name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:5173
void Add_Path_Coord_Attr()
Update the coordinate attribute to include path and also flatten.
Definition: HDF5GMCF.cc:6873
void Handle_Obj_NameClashing(bool)
Handle object name clashing for general NASA HDF5 products.
Definition: HDF5GMCF.cc:4954
void Remove_Unused_FakeDimVars()
Unsupported datatype array may generate FakeDim. Remove them.
Definition: HDF5GMCF.cc:6810
void Update_Product_Type()
Update "product type" attributes for general HDF5 products.
Definition: HDF5GMCF.cc:239
bool Have_Grid_Mapping_Attrs() override
Check if having Grid Mapping Attrs.
Definition: HDF5GMCF.cc:6802
void Handle_DimNameClashing() override
Definition: HDF5GMCF.cc:5055
void Handle_Grid_Mapping_Vars() override
Handle Grid Mapping Vars.
Definition: HDF5GMCF.cc:6806
void Retrieve_H5_CVar_Supported_Attr_Values() override
Retrieve coordinate variable attributes.
Definition: HDF5GMCF.cc:329
void Retrieve_H5_Info(const char *path, hid_t file_id, bool include_attr) override
Retrieve DDS information from the HDF5 file; real implementation for general HDF5 products.
Definition: HDF5GMCF.cc:220
void Handle_SpVar() override
Handle special variables for general NASA HDF5 products.
Definition: HDF5GMCF.cc:4738
const std::string & Get_Ignored_Msg() override
Get the message that contains the ignored obj. info.
Definition: HDF5CF.h:934
void Remove_Unneeded_Objects()
Remove unneeded objects.
Definition: HDF5GMCF.cc:262
void Handle_CVar() override
Handle coordinate variables for general NASA HDF5 products.
Definition: HDF5GMCF.cc:2886
void Add_Dim_Name()
Add dimension name.
Definition: HDF5GMCF.cc:805
void Retrieve_H5_Supported_Attr_Values() override
Retrieve attribute values for the supported HDF5 datatypes for general HDF5 products.
Definition: HDF5GMCF.cc:343
void Handle_Coor_Attr() override
Handle "coordinates" attributes for general HDF5 products.
Definition: HDF5GMCF.cc:5894
void Rename_NC4_NonCoordVars()
Remove the _nc4_non_coord from the variable new names.
Definition: HDF5GMCF.cc:6855
void Handle_Unsupported_Dspace(bool) override
Handle unsupported HDF5 dataspaces for general HDF5 products.
Definition: HDF5GMCF.cc:578
void Handle_Unsupported_Others(bool) override
Handle other unmapped objects/attributes for general HDF5 products.
Definition: HDF5GMCF.cc:668
void Handle_Unsupported_Dtype(bool) override
Handle unsupported HDF5 datatypes for general HDF5 products.
Definition: HDF5GMCF.cc:391
void Flatten_Obj_Name(bool include_attr) override
Flatten the object name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:4905
void Adjust_Dim_Name() override
Adjust dimension name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:5114
void Adjust_Obj_Name() override
Adjust object names based on different general NASA HDF5 products.
Definition: HDF5GMCF.cc:4824
This class is a derived class of Var. It represents a special general HDF5 product(currently ACOS and...
Definition: HDF5CF.h:382
This class represents one HDF5 dataset(CF variable)
Definition: HDF5CF.h:256
float getCompRatio() const
Get the compression ratio of this dataset.
Definition: HDF5CF.h:321
int getRank() const
Get the dimension rank of this variable.
Definition: HDF5CF.h:298
const std::string & getFullPath() const
Get the full path of this variable.
Definition: HDF5CF.h:276
H5DataType getType() const
Get the data type of this variable(Not HDF5 datatype id)
Definition: HDF5CF.h:304
const std::vector< Dimension * > & getDimensions() const
Get the list of the dimensions.
Definition: HDF5CF.h:315
const std::string & getNewName() const
Get the new name of this variable.
Definition: HDF5CF.h:270
Helper functions for generating DAS attributes and a function to check BES Key.
Map and generate DDS and DAS for the CF option for generic HDF5 products.
Type
Type of JSON value.
Definition: rapidjson.h:664